Initial implementation: NATS->S3 archiver + search/retrieve CLI
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
benvin
@@ -0,0 +1,8 @@
|
||||
/dist/
|
||||
*.rpm
|
||||
*.zip
|
||||
# Root-level dev binaries only — anchored so cmd/logarchiver (the source
|
||||
# package) is NOT ignored.
|
||||
/logarchiver
|
||||
# cross-compiled release artifacts (e.g. logarchiver-linux-amd64)
|
||||
/logarchiver-*
|
||||
@@ -0,0 +1,27 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: check-merge-conflict
|
||||
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-fmt
|
||||
- id: go-mod-tidy
|
||||
|
||||
# logarchiver has no root-level Go files (all under cmd/, internal/), so the
|
||||
# dnephin go-vet hook (which runs `go vet` at the repo root) fails with
|
||||
# "no Go files". Vet the whole module instead, mirroring encapi/artifactapi.
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-vet
|
||||
name: go vet
|
||||
entry: go vet ./...
|
||||
language: system
|
||||
types: [go]
|
||||
pass_filenames: false
|
||||
@@ -0,0 +1,18 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- make build
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
@@ -0,0 +1,18 @@
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
steps:
|
||||
- name: docker-logarchiver
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/logarchiver
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
@@ -0,0 +1,18 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
@@ -0,0 +1,142 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -race ./...
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
# Build the dist/ binary (consumed by the RPM step) plus the cross-platform
|
||||
# CLI binaries attached to the Gitea release.
|
||||
- name: build
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- make build VERSION=${CI_COMMIT_TAG}
|
||||
# $$ escapes shell vars so Woodpecker leaves them for the shell instead of
|
||||
# substituting pipeline vars at parse time; ${CI_COMMIT_TAG} is a real var.
|
||||
- |
|
||||
for osarch in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do
|
||||
os="$${osarch%/*}"; arch="$${osarch#*/}"
|
||||
GOOS="$$os" GOARCH="$$arch" \
|
||||
go build -ldflags="-s -w -X main.version=${CI_COMMIT_TAG}" \
|
||||
-o "logarchiver-$${os}-$${arch}" ./cmd/logarchiver
|
||||
done
|
||||
depends_on: [test]
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
# Package the built binary + generated shell completions into an RPM.
|
||||
- name: package
|
||||
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||
commands:
|
||||
- ./scripts/build-rpm.sh ${CI_COMMIT_TAG}
|
||||
depends_on: [build]
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
# Publish the RPM to the artifactapi local rpm repo (a real yum repo;
|
||||
# repodata regenerates automatically).
|
||||
- name: upload-rpm
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260606
|
||||
commands:
|
||||
- |
|
||||
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
REPO="rpm-internal"
|
||||
for rpm in dist/*.rpm; do
|
||||
FILE=$$(basename "$$rpm")
|
||||
# artifactapi has no HEAD route (405); probe with GET against the
|
||||
# served path (RPMs are stored under Packages/) to avoid re-upload.
|
||||
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||
if [ "$$code" = "200" ]; then
|
||||
echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload"
|
||||
continue
|
||||
fi
|
||||
echo "Uploading $$FILE to $$REPO (existence probe returned $$code)"
|
||||
curl -f -X PUT \
|
||||
"$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \
|
||||
-H "Content-Type: application/x-rpm" \
|
||||
--data-binary @"$$rpm"
|
||||
done
|
||||
depends_on: [package]
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
cpu: 500m
|
||||
|
||||
# Cut a Gitea release with the cross-platform CLI binaries attached.
|
||||
- name: release
|
||||
image: git.unkin.net/unkin/almalinux9-base:20260606
|
||||
environment:
|
||||
RELEASER_TOKEN:
|
||||
from_secret: RELEASER_TOKEN
|
||||
commands:
|
||||
- |
|
||||
curl --output /usr/local/bin/tea https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/gitea-dl/tea/0.12.0/tea-0.12.0-linux-amd64 && chmod +x /usr/local/bin/tea
|
||||
tea logins add --name gitea --url https://git.unkin.net --token "$${RELEASER_TOKEN}" --no-version-check
|
||||
# Find the previous release tag for the changelog range; skip tags on the
|
||||
# current commit and pick the newest semver ancestor.
|
||||
CUR_SHA=$$(git rev-list -n1 "${CI_COMMIT_TAG}")
|
||||
PREV_TAG=""
|
||||
for t in $$(git tag --sort=-v:refname); do
|
||||
[ "$$t" = "${CI_COMMIT_TAG}" ] && continue
|
||||
[ "$$(git rev-list -n1 "$$t")" = "$$CUR_SHA" ] && continue
|
||||
if git merge-base --is-ancestor "$$t" "${CI_COMMIT_TAG}" 2>/dev/null; then
|
||||
PREV_TAG="$$t"; break
|
||||
fi
|
||||
done
|
||||
if [ -n "$$PREV_TAG" ]; then
|
||||
NOTES=$$(git log "$${PREV_TAG}..${CI_COMMIT_TAG}" --pretty=format:"- %s")
|
||||
else
|
||||
NOTES=$$(git log --pretty=format:"- %s")
|
||||
fi
|
||||
tea releases create --tag "${CI_COMMIT_TAG}" --title "${CI_COMMIT_TAG}" --note "$${NOTES}" --login gitea --repo "${CI_REPO}"
|
||||
RPM=$$(ls dist/*.rpm 2>/dev/null | head -1)
|
||||
ASSETS="logarchiver-linux-amd64 logarchiver-linux-arm64 logarchiver-darwin-amd64 logarchiver-darwin-arm64"
|
||||
[ -n "$$RPM" ] && ASSETS="$$ASSETS $$RPM"
|
||||
sha256sum $$ASSETS > sha256sums.txt
|
||||
tea releases assets create "${CI_COMMIT_TAG}" $$ASSETS sha256sums.txt \
|
||||
--login gitea --repo "${CI_REPO}"
|
||||
depends_on: [upload-rpm]
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
cpu: 500m
|
||||
@@ -0,0 +1,33 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: lint
|
||||
image: golangci/golangci-lint:latest
|
||||
commands:
|
||||
- golangci-lint run ./...
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -v -race ./...
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 2Gi
|
||||
cpu: 2
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o logarchiver ./cmd/logarchiver
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
|
||||
COPY --from=builder /build/logarchiver /usr/local/bin/logarchiver
|
||||
|
||||
# Prometheus metrics / healthz listener.
|
||||
EXPOSE 9090
|
||||
|
||||
ENTRYPOINT ["logarchiver"]
|
||||
CMD ["run"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Ben Vincent
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,81 @@
|
||||
BINARY := logarchiver
|
||||
PKG := ./cmd/logarchiver
|
||||
DIST := dist
|
||||
MODULE := git.unkin.net/unkin/logarchiver
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||
GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)"
|
||||
OS ?= $(shell go env GOOS)
|
||||
ARCH ?= $(shell go env GOARCH)
|
||||
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
|
||||
.PHONY: all build test test-race lint fmt vet tidy clean install completions rpm rpm-package check-go patch minor major _tag
|
||||
|
||||
all: build
|
||||
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
fi
|
||||
|
||||
# Build the single logarchiver binary (service + CLI) into dist/.
|
||||
build: check-go tidy
|
||||
CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GOFLAGS) -o $(DIST)/$(BINARY) $(PKG)
|
||||
|
||||
test: check-go
|
||||
go test -race -count=1 ./...
|
||||
|
||||
test-race: test
|
||||
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
|
||||
install:
|
||||
go install $(GOFLAGS) $(PKG)
|
||||
|
||||
clean:
|
||||
rm -rf $(DIST) $(BINARY) $(BINARY)-*
|
||||
|
||||
# Generate bash/zsh/fish completions from the freshly built binary.
|
||||
completions: build
|
||||
@mkdir -p $(DIST)/completions
|
||||
$(DIST)/$(BINARY) completion bash > $(DIST)/completions/$(BINARY).bash
|
||||
$(DIST)/$(BINARY) completion zsh > $(DIST)/completions/_$(BINARY)
|
||||
$(DIST)/$(BINARY) completion fish > $(DIST)/completions/$(BINARY).fish
|
||||
|
||||
# Build then package the CLI into an RPM (with completions) via nfpm.
|
||||
rpm: build rpm-package
|
||||
rpm-package:
|
||||
./scripts/build-rpm.sh $(VERSION)
|
||||
|
||||
# Bump helpers — read the latest semver tag and create/push the next one.
|
||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
|
||||
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
|
||||
|
||||
patch:
|
||||
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
|
||||
minor:
|
||||
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
|
||||
major:
|
||||
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
|
||||
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||
|
||||
_tag:
|
||||
git push origin $(TAG)
|
||||
@@ -1,3 +1,80 @@
|
||||
# logarchiver
|
||||
|
||||
Archives raw logs from NATS JetStream to S3 as zstd-compressed, GPG-encrypted, indexed objects; plus a CLI to search and retrieve them. Go service + CLI.
|
||||
Archive raw logs from the centralized logging NATS JetStream stream to S3 as
|
||||
**zstd-compressed, OpenPGP-encrypted, indexed** objects — and a CLI to **search**
|
||||
the index and **retrieve/decrypt** archived logs.
|
||||
|
||||
logarchiver replaces the plain Vector archiver leg of the logging stack
|
||||
(argocd-apps #296). Where Vector wrote gzipped NDJSON to Ceph RGW with no index
|
||||
and no encryption, logarchiver adds:
|
||||
|
||||
- **zstd** compression (`github.com/klauspost/compress/zstd`);
|
||||
- **OpenPGP encryption** with a key held in Ben's Vault GPG secrets engine
|
||||
(`vault-plugin-secrets-gpg`) — the private key never leaves Vault;
|
||||
- a **searchable ClickHouse index** so you can answer *"vault logs from host X
|
||||
between dates Y and Z"* without scanning S3;
|
||||
- **sink-conditional acks**: a batch's JetStream messages are acknowledged only
|
||||
after the object is durably in S3 **and** indexed — something a stock Vector
|
||||
NATS consumer cannot do.
|
||||
|
||||
It is a single Go binary that is both the service (`logarchiver run`) and the
|
||||
operator CLI (`search` / `fetch` / `init-schema`).
|
||||
|
||||
## Quick start
|
||||
|
||||
```sh
|
||||
# Service (in-cluster): drains the JetStream 'archiver' consumer to S3 + index.
|
||||
logarchiver run # defaults suit the logging stack; env supplies secrets
|
||||
|
||||
# Operator CLI (laptop, with VAULT_TOKEN / ~/.vault-token and S3/ClickHouse creds):
|
||||
logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h
|
||||
logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out
|
||||
logarchiver fetch archive/logs.k8s.vault._/2026/07/27/….ndjson.zst.larc -o -
|
||||
|
||||
# Schema (local/dev; in-cluster the argocd bootstrap Job owns this):
|
||||
logarchiver init-schema # or: init-schema --print to emit the DDL
|
||||
```
|
||||
|
||||
## How it works (short version)
|
||||
|
||||
```
|
||||
JetStream LOGS / durable "archiver" ──▶ batch by subject (size/count/time)
|
||||
│ │
|
||||
│ NDJSON ─▶ zstd ─▶ AES-256-GCM frames
|
||||
│ │ (random per-object data key)
|
||||
│ data key ─▶ OpenPGP-encrypt to Vault pubkey
|
||||
│ ▼
|
||||
└── ack ONLY after ───────────────── S3 PUT (LARC1 object) + ClickHouse index row
|
||||
```
|
||||
|
||||
Retrieval sends only the tiny wrapped data key to the Vault GPG engine (which
|
||||
does whole-payload decrypt only), recovers it, and streams the bulk locally.
|
||||
See [docs/architecture.md](docs/architecture.md) and the
|
||||
[retrieval runbook](docs/retrieval-runbook.md) for the full design and the
|
||||
honest account of what the GPG engine supports.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [docs/architecture.md](docs/architecture.md) — components, data flow, the
|
||||
LARC1 container format, index schema, delivery guarantees.
|
||||
- [docs/retrieval-runbook.md](docs/retrieval-runbook.md) — crypto/decrypt design
|
||||
and step-by-step retrieval, key setup, failure modes.
|
||||
- [docs/deployment.md](docs/deployment.md) — drop-in fit with the logging stack
|
||||
(NATS/S3/ClickHouse/Vault wiring, secrets, k8s).
|
||||
- Subcommands: [run](docs/run.md) · [search](docs/search.md) ·
|
||||
[fetch](docs/fetch.md) · [init-schema](docs/init-schema.md)
|
||||
- [config.example.yaml](config.example.yaml) — every config knob with defaults.
|
||||
- [schema/archive_index.sql](schema/archive_index.sql) — the index DDL.
|
||||
|
||||
## Build / test / lint
|
||||
|
||||
```sh
|
||||
make build # -> dist/logarchiver
|
||||
make test # go test -race ./...
|
||||
make lint # golangci-lint run ./...
|
||||
make rpm # build + package the CLI RPM (nfpm)
|
||||
make patch|minor|major# tag + push a release (triggers the Woodpecker pipeline)
|
||||
```
|
||||
|
||||
Releases (on a `v*` tag): a container image `git.unkin.net/unkin/logarchiver`
|
||||
for the service, and a Gitea binary release + `rpm-internal` RPM for the CLI.
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Command logarchiver archives NATS JetStream logs to S3 (zstd + OpenPGP) and
|
||||
// provides a CLI to search the index and retrieve/decrypt archived logs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/cli"
|
||||
)
|
||||
|
||||
// version is injected at build time via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
cli.SetVersion(version)
|
||||
os.Exit(cli.Execute())
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# logarchiver configuration example.
|
||||
#
|
||||
# Every field below shows its built-in default (the defaults are tuned for the
|
||||
# centralized logging stack in argocd-apps #296). In k8s you typically deploy
|
||||
# with NO config file and let the defaults + secret env vars drive everything;
|
||||
# this file documents the knobs and is handy for local/dev runs.
|
||||
#
|
||||
# Precedence: built-in defaults < this file < environment variables.
|
||||
|
||||
nats:
|
||||
url: "nats://nats.logging.svc.cluster.local:4222"
|
||||
stream: "LOGS"
|
||||
# Durable pull-consumer name. Reusing "archiver" takes over the leg the Vector
|
||||
# archiver currently owns; use a distinct name (e.g. "archiver-canary") to run
|
||||
# alongside it during migration.
|
||||
durable: "archiver"
|
||||
# Server-side subject filter(s). Overridable via env ARCHIVE_SUBJECTS
|
||||
# (space-separated). Default archives Vault audit logs only.
|
||||
subjects:
|
||||
- "logs.k8s.vault.>"
|
||||
user: "log-consumer"
|
||||
# Password comes from the nats-auth secret via NATS_CONSUMER_PASSWORD.
|
||||
password: ""
|
||||
password_env: "NATS_CONSUMER_PASSWORD"
|
||||
ca_file: "" # in-cluster NATS is plaintext
|
||||
fetch_batch: 512
|
||||
ack_wait: "2m"
|
||||
|
||||
batch:
|
||||
# A per-subject batch becomes one object when any bound is hit.
|
||||
max_bytes: 67108864 # 64 MiB raw NDJSON
|
||||
max_events: 200000
|
||||
max_age: "5m"
|
||||
|
||||
s3:
|
||||
endpoint: "https://s3.ceph.unkin.net"
|
||||
bucket: "logs-archive"
|
||||
region: "us-east-1"
|
||||
path_style: true
|
||||
# Object key template. Fields: {{.Subject}} {{.Year}} {{.Month}} {{.Day}}.
|
||||
key_prefix: "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/"
|
||||
ca_file: "/etc/vault-ca/ca.crt"
|
||||
# Credentials come from the cephrgw BucketAccess secret logs-archive-s3 via the
|
||||
# standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars. Endpoint/bucket
|
||||
# can also be sourced from that secret's S3_ENDPOINT / BUCKET_NAME keys:
|
||||
endpoint_env: "S3_ENDPOINT"
|
||||
bucket_env: "BUCKET_NAME"
|
||||
|
||||
crypto:
|
||||
# Vault GPG engine key used to encrypt objects (public key) and decrypt on
|
||||
# retrieval (private key, server-side in Vault).
|
||||
key_name: "logarchive"
|
||||
# Where the service gets the PUBLIC key: "file" (mounted armored key, no Vault
|
||||
# dependency for the service) or "vault" (read gpg/keys/<name>).
|
||||
pubkey_source: "file"
|
||||
pubkey_file: "/etc/logarchiver/pubkey.asc"
|
||||
refresh_interval: "1h"
|
||||
frame_size: 1048576 # 1 MiB AES-GCM frames (streaming decrypt granularity)
|
||||
vault:
|
||||
address: "" # falls back to VAULT_ADDR
|
||||
mount: "gpg"
|
||||
auth_method: "kubernetes" # service: kubernetes; CLI always uses token
|
||||
k8s_role: "default"
|
||||
k8s_mount: "k8s/au/syd1"
|
||||
k8s_jwt_path: "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
ca_file: ""
|
||||
|
||||
index:
|
||||
enabled: true
|
||||
address: "clickhouse-logs.logging.svc.cluster.local:9000" # native protocol
|
||||
database: "logs"
|
||||
table: "archive_index"
|
||||
username: "vector"
|
||||
password: ""
|
||||
password_env: "CLICKHOUSE_PASSWORD"
|
||||
tls: false # in-cluster ClickHouse is plaintext
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
address: ":9090" # /metrics and /healthz
|
||||
|
||||
log:
|
||||
level: "info" # debug|info|warn|error
|
||||
format: "json" # json|text
|
||||
@@ -0,0 +1,122 @@
|
||||
# Architecture
|
||||
|
||||
logarchiver is one Go binary with a service mode (`run`) and an operator CLI
|
||||
(`search`/`fetch`/`init-schema`). Boundaries to external systems (NATS, S3,
|
||||
ClickHouse, Vault) sit behind interfaces so they can be faked in tests.
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Responsibility |
|
||||
|---|---|
|
||||
| `internal/config` | Config struct, YAML+env loading, validation, defaults tuned for the logging stack. |
|
||||
| `internal/event` | Extract host + timestamp from a raw JSON event (k8s and VM shapes), subject sanitization. |
|
||||
| `internal/batcher` | Group events per subject into batches; decide flush by size/count/age; carry ack tokens. |
|
||||
| `internal/crypto` | The LARC1 container: zstd + framed AES-256-GCM + OpenPGP-wrapped data key. Seal/Open. |
|
||||
| `internal/vaultgpg` | Thin client for `vault-plugin-secrets-gpg`: fetch public key, decrypt wrapped data key. |
|
||||
| `internal/s3store` | S3/Ceph RGW object storage behind an `ObjectStore` interface. |
|
||||
| `internal/index` | ClickHouse archive index (`Index` interface), DDL, pure search-SQL builder. |
|
||||
| `internal/archiver` | Orchestrates seal → S3 PUT → index write; object-key builder; public-key provider/refresh. |
|
||||
| `internal/consumer` | JetStream binding + the fetch→batch→persist→ack loop (sink-conditional acks). |
|
||||
| `internal/metrics` | Prometheus collectors and `/metrics`. |
|
||||
| `internal/cli` | cobra command tree; wires everything for each subcommand. |
|
||||
|
||||
## Service data flow
|
||||
|
||||
1. **Consume.** A durable JetStream **pull** consumer (`LOGS` stream, durable
|
||||
`archiver`, subject filter default `logs.k8s.vault.>`) is created/updated at
|
||||
startup with explicit acks and independent offsets, so logarchiver's replay
|
||||
position is decoupled from the ClickHouse transform tier. Messages are pulled
|
||||
in batches.
|
||||
2. **Route + batch.** Each message's payload is parsed just enough to extract
|
||||
the source **host** (`.host` → `.hostname` → `.kubernetes.pod_node_name`) and
|
||||
**timestamp** (`.timestamp` → `.ts`). Events are grouped by NATS **subject**.
|
||||
A batch is flushed when it hits `max_bytes` (64 MiB raw), `max_events`, or
|
||||
`max_age` (5 min) — whichever first — or on shutdown.
|
||||
3. **Seal.** The batch is rendered to NDJSON, compressed with zstd, and sealed
|
||||
into a **LARC1** object (see below).
|
||||
4. **Store.** The object is `PUT` to S3 at
|
||||
`archive/<subject>/YYYY/MM/DD/<UTCstamp>-<rand>.ndjson.zst.larc`.
|
||||
5. **Index.** One row per object is inserted into ClickHouse `logs.archive_index`.
|
||||
6. **Ack.** Only now are the batch's JetStream messages acknowledged. If seal,
|
||||
S3, or index fails, the messages are Nak'd (with a backoff) and JetStream
|
||||
redelivers them — nothing is lost on a sink outage.
|
||||
|
||||
## The LARC1 container format
|
||||
|
||||
The Vault GPG engine can only decrypt a **whole** OpenPGP message inline (no
|
||||
session-key extraction, no streaming — see the retrieval runbook). To keep
|
||||
objects any size while making retrieval cheap, logarchiver does its own hybrid
|
||||
encryption instead of PGP-encrypting the whole object:
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------+
|
||||
| magic "LARC1\n" (6 bytes) |
|
||||
| header_len uint32 big-endian |
|
||||
| header JSON { v, key_name, key_fingerprint, |
|
||||
| wrapped_dek_len, nonce_prefix, |
|
||||
| frame_size, compression, cipher } |
|
||||
| wrapped_dek OpenPGP message encrypting the 32-byte DEK | <- only this goes to Vault
|
||||
| to the engine's public key (~hundreds of B) |
|
||||
| frames repeated: uint32 ct_len | ciphertext | <- zstd(NDJSON) in AES-256-GCM
|
||||
| terminated by a zero-length frame | frames; nonce = prefix||counter
|
||||
+-----------------------------------------------------------+
|
||||
```
|
||||
|
||||
- A fresh random 256-bit **DEK** and 4-byte nonce prefix are generated per
|
||||
object. Each frame is AES-256-GCM sealed with nonce `prefix||counter` and AAD
|
||||
= the frame counter (binds frame order; tampering fails the GCM tag).
|
||||
- The DEK is OpenPGP-encrypted to the engine's public key. This wrapped blob is
|
||||
a small, standard OpenPGP message — the only thing ever sent to Vault's
|
||||
decrypt endpoint, regardless of object size.
|
||||
- Compression order is NDJSON → zstd → encrypt, so decryption streams frames,
|
||||
GCM-decrypts, and feeds a streaming zstd decoder to emit NDJSON.
|
||||
|
||||
Because objects are a logarchiver-specific container (not a bare `gpg` file),
|
||||
retrieval must go through `logarchiver fetch`. This is the deliberate cost of
|
||||
supporting arbitrary object sizes against an engine that only whole-payload
|
||||
decrypts.
|
||||
|
||||
## Index schema
|
||||
|
||||
`logs.archive_index` (one row per object), `MergeTree`,
|
||||
`PARTITION BY toYYYYMM(min_ts)`, `ORDER BY (subject, min_ts, object_key)`, with a
|
||||
`bloom_filter` skip index on `hosts`:
|
||||
|
||||
| column | type | notes |
|
||||
|---|---|---|
|
||||
| object_key | String | S3 key |
|
||||
| bucket | LowCardinality(String) | e.g. `logs-archive` |
|
||||
| subject | LowCardinality(String) | NATS subject |
|
||||
| hosts | Array(LowCardinality(String)) | distinct source hosts |
|
||||
| min_ts / max_ts | DateTime64(3) | event time range |
|
||||
| event_count | UInt64 | |
|
||||
| raw_bytes / stored_bytes | UInt64 | pre/post compression+encryption |
|
||||
| compression / cipher / container_format | LowCardinality(String) | `zstd` / `AES-256-GCM` / `LARC1` |
|
||||
| key_name / key_fingerprint | LowCardinality(String) / String | Vault GPG key + 40-hex fingerprint |
|
||||
| created_at | DateTime64(3) DEFAULT now64(3) | |
|
||||
|
||||
Search overlaps the `[from,to]` window (`max_ts >= from AND min_ts <= to`),
|
||||
matches the subject glob with an anchored regex (`*` = one token, `>` = rest),
|
||||
and filters hosts with `has(hosts, …)` (exact) or `arrayExists(… match …)` (glob).
|
||||
|
||||
The DDL is owned in-cluster by the argocd bootstrap Job; `logarchiver
|
||||
init-schema` applies the same statements and `--print` emits them. Keep
|
||||
`schema/archive_index.sql` and `internal/index/ddl.go` in sync.
|
||||
|
||||
## Delivery guarantees
|
||||
|
||||
- **At-least-once**, sink-conditional. Acks happen only after S3 + index
|
||||
success. A failure after S3 but before index leaves an orphan object (still
|
||||
retrievable by prefix; reaped by bucket lifecycle) and triggers redelivery,
|
||||
which produces a fresh object — so an event may appear in two objects, never
|
||||
zero. Downstream consumers of the archive should treat events as at-least-once.
|
||||
- **Graceful shutdown** drains open batches (bounded timeout) so in-flight
|
||||
events are persisted and acked before exit.
|
||||
|
||||
## Configuration & secrets
|
||||
|
||||
Config loads defaults < YAML file < env. Secrets are never in the file:
|
||||
NATS password from `NATS_CONSUMER_PASSWORD`, S3 creds from the standard `AWS_*`
|
||||
env (cephrgw `logs-archive-s3` secret), ClickHouse password from
|
||||
`CLICKHOUSE_PASSWORD`, Vault via `VAULT_ADDR` + k8s auth (service) or ambient
|
||||
token (CLI). See [deployment.md](deployment.md).
|
||||
@@ -0,0 +1,55 @@
|
||||
# Deployment (drop-in fit with the logging stack)
|
||||
|
||||
This describes how logarchiver slots into the centralized logging stack
|
||||
(argocd-apps #296). The actual argocd manifest swap is a **separate later task**;
|
||||
this documents the wiring logarchiver is built for so that swap is mechanical.
|
||||
|
||||
## Where it runs
|
||||
|
||||
- Namespace **`logging`**, ServiceAccount **`default`** (reuses the stack's
|
||||
`VaultAuth` `default`, k8s auth mount `k8s/au/syd1`, role `default`).
|
||||
- Single-replica Deployment (independent JetStream offsets; one archiver is
|
||||
enough — scale by subject-sharding into multiple durables if ever needed).
|
||||
- Pod label **`vector.dev/exclude: "true"`** so the Vector agent does not scrape
|
||||
logarchiver's own logs (matches the Vector archiver it replaces).
|
||||
- Startup healthcheck against RGW should be lenient (RGW cred propagation is
|
||||
slow), like the Vector archiver.
|
||||
|
||||
## What it binds to (all already provided by #296)
|
||||
|
||||
| Dependency | Wiring |
|
||||
|---|---|
|
||||
| **NATS** | `nats://nats.logging.svc.cluster.local:4222`, user `log-consumer`, password from secret **`nats-auth`** key `consumer_password` → env `NATS_CONSUMER_PASSWORD`. Stream `LOGS`, durable `archiver`. |
|
||||
| **S3 (Ceph RGW)** | Bucket `logs-archive`, endpoint `https://s3.ceph.unkin.net` (path-style, region `us-east-1`), CA `/etc/vault-ca/ca.crt` (mount the `vault-ca-cert` secret). Creds from cephrgw BucketAccess secret **`logs-archive-s3`** via `envFrom` (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`; optional `S3_ENDPOINT`/`BUCKET_NAME`). |
|
||||
| **ClickHouse** | `clickhouse-logs.logging.svc.cluster.local:9000` (native), database `logs`, table `archive_index`, user `vector`, password from secret **`clickhouse-credentials`** key `password` → env `CLICKHOUSE_PASSWORD`. |
|
||||
| **Vault GPG engine** | Public key: default `pubkey_source: file` from a mounted armored key (secret/ConfigMap at `/etc/logarchiver/pubkey.asc`) — no Vault dependency on the hot path. Or `pubkey_source: vault` with k8s auth to read `gpg/keys/logarchive`. |
|
||||
|
||||
## Secrets to seed (Vault KV, reusing the `logging/default` path)
|
||||
|
||||
Nothing new is strictly required if you reuse the existing `nats-auth`,
|
||||
`logs-archive-s3`, and `clickhouse-credentials` secrets. For the file pubkey,
|
||||
add an armored public key as a mounted secret (e.g.
|
||||
`kv/kubernetes/namespace/logging/default/logarchiver-pubkey` → VaultStaticSecret
|
||||
→ file mount).
|
||||
|
||||
## Cross-repo notes
|
||||
|
||||
- **No new ServiceAccount** (reuses `default`), so no argocd-apps SA PR is needed
|
||||
for CI; the Woodpecker steps already use `serviceAccountName: default`.
|
||||
- **terraform-vault:** only needed if you (a) use `pubkey_source: vault` and the
|
||||
`logging/default` policy doesn't already permit `read gpg/keys/logarchive`, or
|
||||
(b) want a dedicated operator policy for `update gpg/decrypt/logarchive`.
|
||||
Operators today use their own human Vault tokens for `fetch`, so this is
|
||||
optional — track as a follow-up, not a blocker.
|
||||
- **ClickHouse table:** created by the argocd bootstrap Job (embed the output of
|
||||
`logarchiver init-schema --print`), not by the service at runtime.
|
||||
|
||||
## Migrating off the Vector archiver
|
||||
|
||||
The Vector archiver binds the same `LOGS`/`archiver` durable. To cut over
|
||||
safely: deploy logarchiver with a **distinct** durable (e.g.
|
||||
`durable: archiver-canary`) and a narrow `ARCHIVE_SUBJECTS` to validate objects
|
||||
+ index rows land, then repoint it to the `archiver` durable and scale the
|
||||
Vector archiver to zero. logarchiver writes a different object prefix
|
||||
(`archive/….ndjson.zst.larc`) than Vector (`raw/….log.gz`), so the two never
|
||||
collide in the bucket.
|
||||
@@ -0,0 +1,41 @@
|
||||
# `logarchiver fetch`
|
||||
|
||||
Download, decrypt, and decompress archived objects to plain NDJSON.
|
||||
|
||||
```sh
|
||||
logarchiver fetch [object-key ...] [flags]
|
||||
```
|
||||
|
||||
Objects are selected either by explicit object-key arguments, or — when no keys
|
||||
are given — by the same `--subject/--host/--from/--to` query used by
|
||||
[`search`](search.md). When `--host/--from/--to` are supplied they ALSO
|
||||
re-filter the emitted events to just the matching lines.
|
||||
|
||||
## Flags
|
||||
|
||||
| Flag | Meaning |
|
||||
|---|---|
|
||||
| `-o, --output` | `-` for stdout (default), or a directory to write one NDJSON file per object. |
|
||||
| `--subject`/`--host`/`--from`/`--to`/`--limit` | Object selection (same as `search`) when no keys are given; host/time also re-filter emitted lines. |
|
||||
|
||||
## How decryption works
|
||||
|
||||
For each object, `fetch` reads the object header to learn which Vault GPG key
|
||||
wrapped it, sends only the small wrapped data key to the engine's
|
||||
`gpg/decrypt/<name>` endpoint, recovers the data key, and streams the AES-GCM
|
||||
frames through local decryption + zstd to emit NDJSON. See the
|
||||
[retrieval runbook](retrieval-runbook.md) for the full design and required creds
|
||||
(S3 read, internal CA, ambient `VAULT_TOKEN`/`~/.vault-token`).
|
||||
|
||||
## Examples
|
||||
|
||||
```sh
|
||||
# Straight to stdout by key:
|
||||
logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o -
|
||||
|
||||
# From a query, re-filtered to host db-1 in the last 24h, into ./out:
|
||||
logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out
|
||||
```
|
||||
|
||||
Exit status is non-zero if any selected object fails; per-object errors are
|
||||
reported on stderr and the remaining objects are still processed.
|
||||
@@ -0,0 +1,18 @@
|
||||
# `logarchiver init-schema`
|
||||
|
||||
Create the ClickHouse archive-index database and table (idempotent).
|
||||
|
||||
```sh
|
||||
logarchiver init-schema # execute the DDL against index.address
|
||||
logarchiver init-schema --print # print the DDL instead of executing it
|
||||
```
|
||||
|
||||
In-cluster the argocd bootstrap Job owns schema creation (a ClickHouse PostSync
|
||||
hook, mirroring the logging stack's `clickhouse-schema` job). Use
|
||||
`init-schema --print` to emit the exact `CREATE DATABASE` / `CREATE TABLE`
|
||||
statements to embed in that Job, and `init-schema` (executing) for local/dev.
|
||||
|
||||
The statements are the source-of-truth DDL from `internal/index/ddl.go`, also
|
||||
kept as [`schema/archive_index.sql`](../schema/archive_index.sql). Database and
|
||||
table names come from `index.database` / `index.table` (defaults `logs` /
|
||||
`archive_index`).
|
||||
@@ -0,0 +1,118 @@
|
||||
# Retrieval runbook & crypto design
|
||||
|
||||
This document is the honest account of how encryption and retrieval work, what
|
||||
the Vault GPG engine actually supports, and the operational steps to get plain
|
||||
NDJSON back out of an archived object.
|
||||
|
||||
## What the Vault GPG engine supports (and doesn't)
|
||||
|
||||
logarchiver encrypts to a key in Ben's `vault-plugin-secrets-gpg` engine
|
||||
(transit-style OpenPGP, mounted at `gpg/`). Relevant endpoints:
|
||||
|
||||
- `GET gpg/keys/<name>` → returns `data.public_key` (armored) and
|
||||
`data.fingerprint` (40-hex uppercase, no spaces). Used to encrypt locally.
|
||||
- `POST gpg/decrypt/<name>` with `{ "ciphertext": "<base64 of a whole OpenPGP
|
||||
message>" }` → returns `{ "plaintext": "<base64>" }`.
|
||||
|
||||
**Crucial limitation:** the decrypt endpoint does **whole-payload inline
|
||||
decryption only**. It reads the entire OpenPGP message and returns the entire
|
||||
plaintext. There is **no** session-key (PKESK) extraction, **no** chunking, and
|
||||
**no** streaming. The whole ciphertext must fit in a base64 JSON request body
|
||||
(Vault's default `max_request_size` is 32 MiB), and the whole plaintext comes
|
||||
back base64 in the response. So you cannot feed a large archive object straight
|
||||
to the engine, and you cannot ask it to decrypt only the session key.
|
||||
|
||||
## The design logarchiver chose
|
||||
|
||||
Rather than bound object sizes to the engine's request limit, logarchiver does
|
||||
hybrid encryption itself and sends the engine only a tiny key blob:
|
||||
|
||||
1. **Encrypt (service, local):** generate a random 256-bit **DEK**; encrypt the
|
||||
zstd-compressed NDJSON locally with AES-256-GCM in frames; OpenPGP-encrypt
|
||||
just the 32-byte DEK to the engine's **public** key. The wrapped DEK is a
|
||||
small standard OpenPGP message. The private key is never present.
|
||||
2. **Decrypt (CLI, retrieval):** read the object header, send **only the wrapped
|
||||
DEK** (a few hundred bytes) to `gpg/decrypt/<name>`, get the DEK back, then
|
||||
stream-decrypt the AES-GCM frames locally and zstd-decompress to NDJSON.
|
||||
|
||||
The Vault round-trip is tiny and constant regardless of object size, the engine
|
||||
needs no changes, and the private key stays in Vault. See
|
||||
[architecture.md](architecture.md#the-larc1-container-format) for the container
|
||||
layout.
|
||||
|
||||
> **Possible engine enhancement (follow-up, not required):** if the engine ever
|
||||
> grows a "decrypt session key only / return PKESK plaintext" operation, objects
|
||||
> could be plain standard OpenPGP messages while retaining tiny Vault
|
||||
> round-trips. logarchiver's self-managed DEK envelope already achieves the
|
||||
> operational goal without it, so this is optional.
|
||||
|
||||
## One-time key setup
|
||||
|
||||
Create the archive key in the engine (if it doesn't already exist) and export
|
||||
the public key for the service:
|
||||
|
||||
```sh
|
||||
export VAULT_ADDR=https://vault.service.consul:8200
|
||||
|
||||
# Create the key (idempotent; rsa-3072 default, or ed25519).
|
||||
vault write gpg/keys/logarchive identity="logarchiver <ops@unkin.net>" algorithm=ed25519
|
||||
|
||||
# Fingerprint the service records per object (sanity check):
|
||||
vault read -field=fingerprint gpg/keys/logarchive
|
||||
|
||||
# Export the armored public key for the service's file-mounted pubkey source:
|
||||
vault read -field=public_key gpg/keys/logarchive > pubkey.asc
|
||||
```
|
||||
|
||||
In k8s the service can instead read the public key directly from Vault
|
||||
(`pubkey_source: vault`, k8s auth) — no mounted file needed. The file source is
|
||||
the default because it removes any Vault dependency from the hot path.
|
||||
|
||||
## Retrieve archived logs
|
||||
|
||||
The CLI needs: S3 read creds (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` for the
|
||||
`logs-archive` bucket), the internal CA (`s3.ca_file`), and a Vault token with
|
||||
decrypt on the key (ambient `VAULT_TOKEN` or `~/.vault-token`, exactly like
|
||||
`passv`).
|
||||
|
||||
```sh
|
||||
export VAULT_ADDR=https://vault.service.consul:8200
|
||||
vault login ... # or have ~/.vault-token
|
||||
export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=...
|
||||
|
||||
# 1. Find objects via the index.
|
||||
logarchiver search --subject 'logs.k8s.vault.>' --host node-1 \
|
||||
--from 2026-07-01 --to 2026-07-27
|
||||
|
||||
# 2a. Retrieve by the object keys search printed, to stdout:
|
||||
logarchiver fetch <object-key> [<object-key> ...] -o -
|
||||
|
||||
# 2b. Or retrieve straight from a query, re-filtered to just the matching
|
||||
# host/time events, into a directory (one NDJSON file per object):
|
||||
logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out
|
||||
```
|
||||
|
||||
`fetch` downloads each object, reads its header to learn which Vault key wrapped
|
||||
it (`key_name`), decrypts the wrapped DEK via the engine, streams the frames
|
||||
through local AES-GCM + zstd, and emits NDJSON. When `--host`/`--from`/`--to`
|
||||
are supplied they also re-filter the emitted lines (events lacking a parseable
|
||||
timestamp are kept, never silently dropped on time grounds).
|
||||
|
||||
## Failure modes
|
||||
|
||||
| Symptom | Likely cause | Action |
|
||||
|---|---|---|
|
||||
| `decrypt via gpg/decrypt/<name>: permission denied` | Token lacks the decrypt policy | Grant `update` on `gpg/decrypt/<name>` (and `read` on `gpg/keys/<name>`). |
|
||||
| `unwrap dek: … no plaintext` | Wrong key name / key rotated below min_decryption_version | Confirm object `key_name`; the engine tries versions ≥ min_decryption_version. |
|
||||
| `bad magic: not a logarchiver (LARC1) object` | Fetching a non-logarchiver object (e.g. an old Vector `.log.gz`) | Use the right prefix; Vector objects are plain gzip, decrypt them with gzip. |
|
||||
| `decrypt frame N: cipher: message authentication failed` | Object corruption/tampering | Object integrity is broken; re-archive from JetStream if still within retention. |
|
||||
| `pubkey fingerprint mismatch` (service) | Armored pubkey doesn't match the engine's reported fingerprint | Re-export the pubkey; the service refuses to encrypt to a mismatched key. |
|
||||
|
||||
## Key rotation
|
||||
|
||||
Rotating the engine key (`vault write gpg/keys/logarchive/rotate ...`) starts
|
||||
encrypting new objects to the new version; the service picks up the new public
|
||||
key within `crypto.refresh_interval` (default 1h) or on restart. Old objects
|
||||
remain decryptable because the engine tries every key version from
|
||||
`min_decryption_version` up. Each object records the fingerprint it was sealed
|
||||
with, so you can always tell which key version applies.
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# `logarchiver run`
|
||||
|
||||
Runs the archiver service: binds the JetStream pull consumer and drains it to
|
||||
S3 + the ClickHouse index, acking only after each object is durably persisted.
|
||||
|
||||
```sh
|
||||
logarchiver run [-c config.yaml]
|
||||
```
|
||||
|
||||
## Behaviour
|
||||
|
||||
- Creates/updates the durable consumer (`nats.stream` / `nats.durable`) with the
|
||||
configured subject filters, explicit acks, and `nats.ack_wait`.
|
||||
- Batches events per subject and flushes on `batch.max_bytes` / `max_events` /
|
||||
`max_age`. Each flush seals a LARC1 object, PUTs it to S3, writes one index
|
||||
row, then acks the batch. On any failure the batch is Nak'd for redelivery.
|
||||
- Loads the OpenPGP public key from `crypto.pubkey_source` (`file` or `vault`)
|
||||
and refreshes it every `crypto.refresh_interval`.
|
||||
- Serves Prometheus metrics and `/healthz` on `metrics.address` (default `:9090`)
|
||||
when `metrics.enabled`.
|
||||
- Handles SIGINT/SIGTERM: stops fetching, drains open batches (bounded), exits.
|
||||
|
||||
## Key env vars
|
||||
|
||||
| Env | Purpose |
|
||||
|---|---|
|
||||
| `NATS_CONSUMER_PASSWORD` | NATS `log-consumer` password (nats-auth secret). |
|
||||
| `ARCHIVE_SUBJECTS` | Space-separated subject filters (overrides `nats.subjects`). |
|
||||
| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | S3 creds (cephrgw `logs-archive-s3`). |
|
||||
| `S3_ENDPOINT` / `BUCKET_NAME` | Optional S3 endpoint/bucket from the cephrgw secret. |
|
||||
| `CLICKHOUSE_PASSWORD` | ClickHouse `vector` user password. |
|
||||
| `VAULT_ADDR` | Vault address when `pubkey_source: vault`. |
|
||||
| `LOGARCHIVER_CONFIG` | Config file path (same as `-c`). |
|
||||
|
||||
## Metrics
|
||||
|
||||
`logarchiver_objects_stored_total`, `_events_archived_total`,
|
||||
`_raw_bytes_total`, `_stored_bytes_total`, `_store_failures_total`,
|
||||
`_index_failures_total`, `_messages_fetched_total`, `_acks_total`,
|
||||
`_batches_flushed_total{trigger}`, `logarchiver_pending_events`.
|
||||
@@ -0,0 +1,39 @@
|
||||
# `logarchiver search`
|
||||
|
||||
Query the ClickHouse archive index for objects matching a subject/host/time
|
||||
window. Prints one row per matching S3 object with event counts and sizes; use
|
||||
the object keys with [`fetch`](fetch.md).
|
||||
|
||||
```sh
|
||||
logarchiver search [flags]
|
||||
```
|
||||
|
||||
## Flags
|
||||
|
||||
| Flag | Meaning |
|
||||
|---|---|
|
||||
| `--subject` | NATS-style subject glob: `*` matches one token, `>` matches the rest. e.g. `logs.vm.*`, `logs.k8s.vault.>`. |
|
||||
| `--host` | Source host to match: exact, or a glob containing `*`. |
|
||||
| `--from` | Start of window: RFC3339, `YYYY-MM-DD`, or a relative duration like `-24h`. |
|
||||
| `--to` | End of window (same formats). |
|
||||
| `--limit` | Max objects (default 100; `0` = no limit). |
|
||||
| `--json` | Emit results as JSON instead of a table. |
|
||||
|
||||
An object matches the time window when its `[min_ts, max_ts]` overlaps
|
||||
`[from, to]`. Objects are ordered by `min_ts`.
|
||||
|
||||
## Examples
|
||||
|
||||
```sh
|
||||
# Vault audit logs from a node in the last day.
|
||||
logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h
|
||||
|
||||
# All VM logs for a host in July, as JSON.
|
||||
logarchiver search --subject 'logs.vm.*' --host db-1 \
|
||||
--from 2026-07-01 --to 2026-08-01 --json
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
Uses `index.*` (ClickHouse address/database/table/credentials). Requires
|
||||
`index.enabled: true`.
|
||||
@@ -0,0 +1,78 @@
|
||||
module git.unkin.net/unkin/logarchiver
|
||||
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.47.0
|
||||
github.com/ProtonMail/go-crypto v1.4.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.31
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/klauspost/compress v1.19.1
|
||||
github.com/nats-io/nats.go v1.52.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/spf13/cobra v1.10.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/ch-go v0.73.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect
|
||||
github.com/aws/smithy-go v1.27.3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.2 // indirect
|
||||
github.com/go-faster/city v1.0.1 // indirect
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/paulmach/orb v0.13.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.27 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/segmentio/asm v1.2.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
@@ -0,0 +1,169 @@
|
||||
github.com/ClickHouse/ch-go v0.73.0 h1:jsHiGRbQ3sz+gekvDFJF29LWDo5dzbJm5s1h8TWVP2M=
|
||||
github.com/ClickHouse/ch-go v0.73.0/go.mod h1:wkFIxrqlXeRJ9cn3r5Fz5Qen9jl5aTMPuGZeuJpANNY=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.47.0 h1:ZDAzrnKSOPTIsm4tdUNfrii2yc8dk4SVRLC77BR7Z5Q=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.47.0/go.mod h1:sPj7C7UYQ2MWHcfX+4eGN6nwnCqwUKfgO6PcwKpd6K8=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg=
|
||||
github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
|
||||
github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudflare/circl v1.6.2 h1:hL7VBpHHKzrV5WTfHCaBsgx/HGbBYlgrwvNXEVDYYsQ=
|
||||
github.com/cloudflare/circl v1.6.2/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
|
||||
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
|
||||
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
|
||||
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM=
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
|
||||
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I=
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
|
||||
github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
|
||||
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
|
||||
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=
|
||||
github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=
|
||||
github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
|
||||
github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
|
||||
github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,161 @@
|
||||
// Package archiver turns a ready batch into a stored, indexed object: build
|
||||
// NDJSON, seal it into a LARC1 container, PUT it to S3, then write the index
|
||||
// row. Store returns an error if ANY step fails; the caller must not ack the
|
||||
// batch's messages until Store succeeds (at-least-once, sink-conditional acks).
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
||||
"git.unkin.net/unkin/logarchiver/internal/crypto"
|
||||
"git.unkin.net/unkin/logarchiver/internal/index"
|
||||
"git.unkin.net/unkin/logarchiver/internal/s3store"
|
||||
)
|
||||
|
||||
// Metrics is the optional metrics sink (implemented by internal/metrics). A nil
|
||||
// Metrics is fine (no-op).
|
||||
type Metrics interface {
|
||||
ObjectStored(subject string, events int, rawBytes, storedBytes int64)
|
||||
StoreFailed(subject string)
|
||||
IndexFailed(subject string)
|
||||
}
|
||||
|
||||
// Archiver persists batches.
|
||||
type Archiver struct {
|
||||
keys *KeyBuilder
|
||||
pubkeys *PubkeyProvider
|
||||
store s3store.ObjectStore
|
||||
idx index.Index // may be nil when indexing is disabled
|
||||
keyName string
|
||||
frameSize int
|
||||
metrics Metrics
|
||||
nowFn func() time.Time
|
||||
}
|
||||
|
||||
// Options configures an Archiver.
|
||||
type Options struct {
|
||||
Keys *KeyBuilder
|
||||
Pubkeys *PubkeyProvider
|
||||
Store s3store.ObjectStore
|
||||
Index index.Index
|
||||
KeyName string
|
||||
FrameSize int
|
||||
Metrics Metrics
|
||||
}
|
||||
|
||||
// New builds an Archiver.
|
||||
func New(o Options) (*Archiver, error) {
|
||||
if o.Keys == nil || o.Pubkeys == nil || o.Store == nil {
|
||||
return nil, fmt.Errorf("archiver requires keys, pubkeys and store")
|
||||
}
|
||||
fs := o.FrameSize
|
||||
if fs <= 0 {
|
||||
fs = 1 << 20
|
||||
}
|
||||
return &Archiver{
|
||||
keys: o.Keys,
|
||||
pubkeys: o.Pubkeys,
|
||||
store: o.Store,
|
||||
idx: o.Index,
|
||||
keyName: o.KeyName,
|
||||
frameSize: fs,
|
||||
metrics: o.Metrics,
|
||||
nowFn: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StoreResult reports what Store persisted.
|
||||
type StoreResult struct {
|
||||
ObjectKey string
|
||||
Events int
|
||||
RawBytes int64
|
||||
StoredBytes int64
|
||||
}
|
||||
|
||||
// Store seals, uploads and indexes a batch. On success the caller may ack.
|
||||
func (a *Archiver) Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) {
|
||||
if len(batch.Items) == 0 {
|
||||
return StoreResult{}, nil
|
||||
}
|
||||
now := a.nowFn().UTC()
|
||||
summary := batch.Summarize(now)
|
||||
|
||||
pub := a.pubkeys.Current()
|
||||
if pub == nil {
|
||||
a.metricStoreFailed(batch.Subject)
|
||||
return StoreResult{}, fmt.Errorf("no public key available")
|
||||
}
|
||||
|
||||
// Choose the object key from the batch's max timestamp so it lands in the
|
||||
// date partition of the newest event.
|
||||
key, err := a.keys.Build(batch.Subject, summary.MaxTS)
|
||||
if err != nil {
|
||||
a.metricStoreFailed(batch.Subject)
|
||||
return StoreResult{}, err
|
||||
}
|
||||
|
||||
ndjson := batch.NDJSON()
|
||||
var buf bytes.Buffer
|
||||
sealed, err := crypto.Seal(&buf, ndjson, pub, a.keyName, a.frameSize)
|
||||
if err != nil {
|
||||
a.metricStoreFailed(batch.Subject)
|
||||
return StoreResult{}, fmt.Errorf("seal object %s: %w", key, err)
|
||||
}
|
||||
|
||||
if err := a.store.Put(ctx, key, bytes.NewReader(buf.Bytes()), int64(buf.Len())); err != nil {
|
||||
a.metricStoreFailed(batch.Subject)
|
||||
return StoreResult{}, err
|
||||
}
|
||||
|
||||
if a.idx != nil {
|
||||
row := index.Row{
|
||||
ObjectKey: key,
|
||||
Bucket: a.store.Bucket(),
|
||||
Subject: batch.Subject,
|
||||
Hosts: summary.Hosts,
|
||||
MinTS: summary.MinTS,
|
||||
MaxTS: summary.MaxTS,
|
||||
EventCount: uint64(summary.EventCount),
|
||||
RawBytes: uint64(sealed.RawBytes),
|
||||
StoredBytes: uint64(sealed.StoredBytes),
|
||||
Compression: sealed.Header.Compression,
|
||||
Cipher: sealed.Header.Cipher,
|
||||
ContainerFormat: "LARC1",
|
||||
KeyName: a.keyName,
|
||||
KeyFingerprint: sealed.Header.KeyFingerprint,
|
||||
}
|
||||
if err := a.idx.Insert(ctx, row); err != nil {
|
||||
// The object is in S3 but unindexed. Do NOT ack: on redelivery the
|
||||
// batch is re-stored (a new object key) and re-indexed. The orphan
|
||||
// object is harmless (retrievable by prefix) and reaped by lifecycle.
|
||||
a.metricIndexFailed(batch.Subject)
|
||||
return StoreResult{}, fmt.Errorf("index object %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
if a.metrics != nil {
|
||||
a.metrics.ObjectStored(batch.Subject, summary.EventCount, sealed.RawBytes, sealed.StoredBytes)
|
||||
}
|
||||
return StoreResult{
|
||||
ObjectKey: key,
|
||||
Events: summary.EventCount,
|
||||
RawBytes: sealed.RawBytes,
|
||||
StoredBytes: sealed.StoredBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Archiver) metricStoreFailed(subject string) {
|
||||
if a.metrics != nil {
|
||||
a.metrics.StoreFailed(subject)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Archiver) metricIndexFailed(subject string) {
|
||||
if a.metrics != nil {
|
||||
a.metrics.IndexFailed(subject)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
||||
"git.unkin.net/unkin/logarchiver/internal/crypto"
|
||||
"git.unkin.net/unkin/logarchiver/internal/index"
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
)
|
||||
|
||||
// --- fakes ---
|
||||
|
||||
type fakeStore struct {
|
||||
bucket string
|
||||
objects map[string][]byte
|
||||
failPut bool
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore {
|
||||
return &fakeStore{bucket: "test-bucket", objects: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func (f *fakeStore) Put(_ context.Context, key string, body io.Reader, _ int64) error {
|
||||
if f.failPut {
|
||||
return errors.New("simulated s3 failure")
|
||||
}
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.objects[key] = data
|
||||
return nil
|
||||
}
|
||||
func (f *fakeStore) Get(_ context.Context, key string) (io.ReadCloser, error) {
|
||||
data, ok := f.objects[key]
|
||||
if !ok {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
func (f *fakeStore) List(_ context.Context, _ string) ([]string, error) { return nil, nil }
|
||||
func (f *fakeStore) Bucket() string { return f.bucket }
|
||||
|
||||
type fakeIndex struct {
|
||||
rows []index.Row
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (f *fakeIndex) Insert(_ context.Context, row index.Row) error {
|
||||
if f.fail {
|
||||
return errors.New("simulated index failure")
|
||||
}
|
||||
f.rows = append(f.rows, row)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeIndex) Search(context.Context, index.SearchQuery) ([]index.Result, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeIndex) InitSchema(context.Context) error { return nil }
|
||||
func (f *fakeIndex) Ping(context.Context) error { return nil }
|
||||
func (f *fakeIndex) Close() error { return nil }
|
||||
|
||||
func testPubkey(t *testing.T) *crypto.PublicKey {
|
||||
t.Helper()
|
||||
ent, err := openpgp.NewEntity("t", "", "t@unkin.net", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEntity: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, _ := armor.Encode(&buf, openpgp.PublicKeyType, nil)
|
||||
_ = ent.Serialize(w)
|
||||
_ = w.Close()
|
||||
pk, err := crypto.LoadPublicKey(buf.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublicKey: %v", err)
|
||||
}
|
||||
return pk
|
||||
}
|
||||
|
||||
func newTestArchiver(t *testing.T, store *fakeStore, idx index.Index) *Archiver {
|
||||
t.Helper()
|
||||
pk := testPubkey(t)
|
||||
kb, _ := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/")
|
||||
prov, err := NewPubkeyProvider(context.Background(), func(context.Context) (*crypto.PublicKey, error) {
|
||||
return pk, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("provider: %v", err)
|
||||
}
|
||||
a, err := New(Options{
|
||||
Keys: kb,
|
||||
Pubkeys: prov,
|
||||
Store: store,
|
||||
Index: idx,
|
||||
KeyName: "logarchive",
|
||||
FrameSize: 4096,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func sampleBatch() *batcher.Batch {
|
||||
ts := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
|
||||
return &batcher.Batch{
|
||||
Subject: "logs.k8s.vault.audit",
|
||||
Items: []batcher.Item{
|
||||
{Subject: "logs.k8s.vault.audit", Host: "node-1", Timestamp: ts, HasTS: true, Raw: []byte(`{"host":"node-1","message":"a"}`)},
|
||||
{Subject: "logs.k8s.vault.audit", Host: "node-2", Timestamp: ts.Add(time.Hour), HasTS: true, Raw: []byte(`{"host":"node-2","message":"b"}`)},
|
||||
},
|
||||
RawBytes: 62,
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreSuccessWritesObjectAndIndex(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
idx := &fakeIndex{}
|
||||
a := newTestArchiver(t, store, idx)
|
||||
|
||||
res, err := a.Store(context.Background(), sampleBatch())
|
||||
if err != nil {
|
||||
t.Fatalf("Store: %v", err)
|
||||
}
|
||||
if res.Events != 2 {
|
||||
t.Errorf("events = %d", res.Events)
|
||||
}
|
||||
if len(store.objects) != 1 {
|
||||
t.Fatalf("expected 1 stored object, got %d", len(store.objects))
|
||||
}
|
||||
// Stored bytes must be a valid LARC1 container.
|
||||
obj := store.objects[res.ObjectKey]
|
||||
if _, _, err := crypto.ReadHeader(bytes.NewReader(obj)); err != nil {
|
||||
t.Errorf("stored object is not a valid container: %v", err)
|
||||
}
|
||||
if len(idx.rows) != 1 {
|
||||
t.Fatalf("expected 1 index row, got %d", len(idx.rows))
|
||||
}
|
||||
row := idx.rows[0]
|
||||
if row.Subject != "logs.k8s.vault.audit" {
|
||||
t.Errorf("row subject = %q", row.Subject)
|
||||
}
|
||||
if row.EventCount != 2 {
|
||||
t.Errorf("row event_count = %d", row.EventCount)
|
||||
}
|
||||
if len(row.Hosts) != 2 || row.Hosts[0] != "node-1" || row.Hosts[1] != "node-2" {
|
||||
t.Errorf("row hosts = %v", row.Hosts)
|
||||
}
|
||||
if row.Bucket != "test-bucket" {
|
||||
t.Errorf("row bucket = %q", row.Bucket)
|
||||
}
|
||||
if row.ContainerFormat != "LARC1" || row.Compression != "zstd" {
|
||||
t.Errorf("row metadata wrong: %+v", row)
|
||||
}
|
||||
if row.KeyFingerprint == "" {
|
||||
t.Errorf("row missing key fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
// The central at-least-once property: if S3 fails, Store errors and NOTHING is
|
||||
// written to the index, so the caller will not ack.
|
||||
func TestStoreS3FailureNoIndexNoAck(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.failPut = true
|
||||
idx := &fakeIndex{}
|
||||
a := newTestArchiver(t, store, idx)
|
||||
|
||||
if _, err := a.Store(context.Background(), sampleBatch()); err == nil {
|
||||
t.Fatalf("expected error when S3 put fails")
|
||||
}
|
||||
if len(idx.rows) != 0 {
|
||||
t.Errorf("index written despite S3 failure: %d rows", len(idx.rows))
|
||||
}
|
||||
if len(store.objects) != 0 {
|
||||
t.Errorf("object recorded despite S3 failure")
|
||||
}
|
||||
}
|
||||
|
||||
// If indexing fails after the S3 put, Store still errors (so no ack); the object
|
||||
// exists but is unindexed — acceptable, it will be re-stored on redelivery.
|
||||
func TestStoreIndexFailureErrors(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
idx := &fakeIndex{fail: true}
|
||||
a := newTestArchiver(t, store, idx)
|
||||
|
||||
if _, err := a.Store(context.Background(), sampleBatch()); err == nil {
|
||||
t.Fatalf("expected error when index insert fails")
|
||||
}
|
||||
if len(store.objects) != 1 {
|
||||
t.Errorf("object should still be in S3 (orphan), got %d", len(store.objects))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreNilIndexOK(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
a := newTestArchiver(t, store, nil)
|
||||
if _, err := a.Store(context.Background(), sampleBatch()); err != nil {
|
||||
t.Fatalf("Store with nil index: %v", err)
|
||||
}
|
||||
if len(store.objects) != 1 {
|
||||
t.Errorf("expected object stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreEmptyBatchNoop(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
a := newTestArchiver(t, store, &fakeIndex{})
|
||||
res, err := a.Store(context.Background(), &batcher.Batch{Subject: "s"})
|
||||
if err != nil {
|
||||
t.Fatalf("empty batch: %v", err)
|
||||
}
|
||||
if res.ObjectKey != "" || len(store.objects) != 0 {
|
||||
t.Errorf("empty batch should store nothing")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/event"
|
||||
)
|
||||
|
||||
// ObjectExt is the suffix for logarchiver container objects (zstd + framed
|
||||
// AES-GCM + wrapped OpenPGP DEK). It is deliberately NOT .gz/.pgp because the
|
||||
// object is a logarchiver-specific container, not a bare gpg file.
|
||||
const ObjectExt = ".ndjson.zst.larc"
|
||||
|
||||
// KeyBuilder renders S3 object keys from a prefix template. Template fields:
|
||||
// {{.Subject}} (sanitized), {{.Year}} {{.Month}} {{.Day}} (UTC, zero-padded).
|
||||
type KeyBuilder struct {
|
||||
tmpl *template.Template
|
||||
}
|
||||
|
||||
// NewKeyBuilder compiles the prefix template.
|
||||
func NewKeyBuilder(prefixTemplate string) (*KeyBuilder, error) {
|
||||
t, err := template.New("key").Option("missingkey=error").Parse(prefixTemplate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse key_prefix template: %w", err)
|
||||
}
|
||||
return &KeyBuilder{tmpl: t}, nil
|
||||
}
|
||||
|
||||
type keyData struct {
|
||||
Subject string
|
||||
Year string
|
||||
Month string
|
||||
Day string
|
||||
}
|
||||
|
||||
// Build returns a unique object key for a batch of subject at ts. The filename
|
||||
// is <UTCstamp>-<random>.ndjson.zst.larc so keys are collision-free and sortable.
|
||||
func (k *KeyBuilder) Build(subject string, ts time.Time) (string, error) {
|
||||
ts = ts.UTC()
|
||||
var sb strings.Builder
|
||||
err := k.tmpl.Execute(&sb, keyData{
|
||||
Subject: event.SubjectToken(subject),
|
||||
Year: fmt.Sprintf("%04d", ts.Year()),
|
||||
Month: fmt.Sprintf("%02d", int(ts.Month())),
|
||||
Day: fmt.Sprintf("%02d", ts.Day()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("render key prefix: %w", err)
|
||||
}
|
||||
prefix := sb.String()
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
suffix, err := randHex(8)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
name := ts.Format("20060102T150405Z") + "-" + suffix + ObjectExt
|
||||
return prefix + name, nil
|
||||
}
|
||||
|
||||
func randHex(n int) (string, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", fmt.Errorf("random: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestKeyBuilder(t *testing.T) {
|
||||
kb, err := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/")
|
||||
if err != nil {
|
||||
t.Fatalf("NewKeyBuilder: %v", err)
|
||||
}
|
||||
ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
|
||||
key, err := kb.Build("logs.k8s.vault.audit", ts)
|
||||
if err != nil {
|
||||
t.Fatalf("Build: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(key, "archive/logs.k8s.vault.audit/2026/07/05/") {
|
||||
t.Errorf("unexpected prefix: %s", key)
|
||||
}
|
||||
if !strings.HasSuffix(key, ObjectExt) {
|
||||
t.Errorf("missing container suffix: %s", key)
|
||||
}
|
||||
if !strings.Contains(key, "20260705T101500Z-") {
|
||||
t.Errorf("missing timestamp token: %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyBuilderUnique(t *testing.T) {
|
||||
kb, _ := NewKeyBuilder("p/{{.Subject}}/")
|
||||
ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
|
||||
k1, _ := kb.Build("s", ts)
|
||||
k2, _ := kb.Build("s", ts)
|
||||
if k1 == k2 {
|
||||
t.Errorf("keys should be unique: %s", k1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyBuilderSanitizesSubject(t *testing.T) {
|
||||
kb, _ := NewKeyBuilder("p/{{.Subject}}/")
|
||||
key, _ := kb.Build("logs.k8s.a/b", time.Now())
|
||||
if strings.Contains(key, "a/b") {
|
||||
t.Errorf("subject slash not sanitized: %s", key)
|
||||
}
|
||||
if !strings.Contains(key, "a_b") {
|
||||
t.Errorf("expected sanitized a_b: %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyBuilderBadTemplate(t *testing.T) {
|
||||
if _, err := NewKeyBuilder("{{.Nope"); err == nil {
|
||||
t.Errorf("expected template parse error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/config"
|
||||
"git.unkin.net/unkin/logarchiver/internal/crypto"
|
||||
"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
|
||||
)
|
||||
|
||||
// PubkeyLoader fetches the current armored public key from the configured source.
|
||||
type PubkeyLoader func(ctx context.Context) (*crypto.PublicKey, error)
|
||||
|
||||
// PubkeyProvider caches the current public key and supports periodic refresh so
|
||||
// key rotation in the Vault GPG engine is picked up without a restart.
|
||||
type PubkeyProvider struct {
|
||||
load PubkeyLoader
|
||||
mu sync.RWMutex
|
||||
key *crypto.PublicKey
|
||||
}
|
||||
|
||||
// NewPubkeyProvider builds a provider and loads the key once.
|
||||
func NewPubkeyProvider(ctx context.Context, load PubkeyLoader) (*PubkeyProvider, error) {
|
||||
p := &PubkeyProvider{load: load}
|
||||
if err := p.Refresh(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Refresh reloads the public key.
|
||||
func (p *PubkeyProvider) Refresh(ctx context.Context) error {
|
||||
key, err := p.load(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.key = key
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Current returns the cached public key.
|
||||
func (p *PubkeyProvider) Current() *crypto.PublicKey {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.key
|
||||
}
|
||||
|
||||
// PubkeyLoaderFromConfig builds a loader for the configured source. For
|
||||
// pubkey_source=vault it also verifies the parsed key's fingerprint matches the
|
||||
// fingerprint the engine reports, catching armor corruption.
|
||||
func PubkeyLoaderFromConfig(cfg config.CryptoConfig, vc *vaultgpg.Client) (PubkeyLoader, error) {
|
||||
switch cfg.Source {
|
||||
case config.PubkeyFile:
|
||||
path := cfg.PubkeyFile
|
||||
return func(_ context.Context) (*crypto.PublicKey, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read pubkey file %s: %w", path, err)
|
||||
}
|
||||
return crypto.LoadPublicKey(data)
|
||||
}, nil
|
||||
case config.PubkeyVault:
|
||||
if vc == nil {
|
||||
return nil, fmt.Errorf("pubkey_source=vault requires a vault client")
|
||||
}
|
||||
name := cfg.KeyName
|
||||
return func(ctx context.Context) (*crypto.PublicKey, error) {
|
||||
pk, err := vc.FetchPublicKey(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := crypto.LoadPublicKey([]byte(pk.Armored))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pk.Fingerprint != "" && key.Fingerprint != pk.Fingerprint {
|
||||
return nil, fmt.Errorf("pubkey fingerprint mismatch: engine=%s parsed=%s", pk.Fingerprint, key.Fingerprint)
|
||||
}
|
||||
return key, nil
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown pubkey_source %q", cfg.Source)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Package batcher groups incoming log events into per-subject batches and
|
||||
// decides when a batch is ready to become one archived object. It is
|
||||
// deliberately not concurrent: the consumer run loop owns a Batcher and drives
|
||||
// it from a single goroutine (Add on receive, DueByAge on a ticker, Drain on
|
||||
// shutdown), which keeps the ack-after-persist accounting simple and race-free.
|
||||
package batcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one log event routed into a batch. Ack is an opaque token (a
|
||||
// jetstream.Msg in production) that the caller acknowledges only after the batch
|
||||
// has been durably persisted.
|
||||
type Item struct {
|
||||
Subject string
|
||||
Raw []byte
|
||||
Host string
|
||||
Timestamp time.Time
|
||||
HasTS bool
|
||||
Ack any
|
||||
}
|
||||
|
||||
// Limits bound a single batch/object.
|
||||
type Limits struct {
|
||||
MaxBytes int64
|
||||
MaxEvents int
|
||||
MaxAge time.Duration
|
||||
}
|
||||
|
||||
// Batch is a ready (or in-progress) group of events for one subject.
|
||||
type Batch struct {
|
||||
Subject string
|
||||
Items []Item
|
||||
RawBytes int64
|
||||
OpenedAt time.Time
|
||||
}
|
||||
|
||||
// Batcher accumulates open batches keyed by subject.
|
||||
type Batcher struct {
|
||||
limits Limits
|
||||
open map[string]*Batch
|
||||
nowFn func() time.Time
|
||||
}
|
||||
|
||||
// New returns a Batcher enforcing limits.
|
||||
func New(limits Limits) *Batcher {
|
||||
return &Batcher{limits: limits, open: map[string]*Batch{}, nowFn: time.Now}
|
||||
}
|
||||
|
||||
// Add appends it to its subject's open batch. If that batch is now full (by
|
||||
// bytes or event count), it is removed from the open set and returned so the
|
||||
// caller can flush it; otherwise Add returns nil.
|
||||
func (b *Batcher) Add(it Item) *Batch {
|
||||
batch := b.open[it.Subject]
|
||||
if batch == nil {
|
||||
batch = &Batch{Subject: it.Subject, OpenedAt: b.nowFn()}
|
||||
b.open[it.Subject] = batch
|
||||
}
|
||||
batch.Items = append(batch.Items, it)
|
||||
batch.RawBytes += int64(len(it.Raw))
|
||||
|
||||
if b.full(batch) {
|
||||
delete(b.open, it.Subject)
|
||||
return batch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batcher) full(batch *Batch) bool {
|
||||
if b.limits.MaxBytes > 0 && batch.RawBytes >= b.limits.MaxBytes {
|
||||
return true
|
||||
}
|
||||
if b.limits.MaxEvents > 0 && len(batch.Items) >= b.limits.MaxEvents {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DueByAge removes and returns every open batch older than MaxAge as of now.
|
||||
func (b *Batcher) DueByAge(now time.Time) []*Batch {
|
||||
if b.limits.MaxAge <= 0 {
|
||||
return nil
|
||||
}
|
||||
var due []*Batch
|
||||
for subj, batch := range b.open {
|
||||
if now.Sub(batch.OpenedAt) >= b.limits.MaxAge {
|
||||
due = append(due, batch)
|
||||
delete(b.open, subj)
|
||||
}
|
||||
}
|
||||
sortBatches(due)
|
||||
return due
|
||||
}
|
||||
|
||||
// Drain removes and returns all open batches (used on graceful shutdown so
|
||||
// in-flight events are persisted and acked before exit).
|
||||
func (b *Batcher) Drain() []*Batch {
|
||||
var all []*Batch
|
||||
for subj, batch := range b.open {
|
||||
all = append(all, batch)
|
||||
delete(b.open, subj)
|
||||
}
|
||||
sortBatches(all)
|
||||
return all
|
||||
}
|
||||
|
||||
// Pending reports how many events sit in open batches.
|
||||
func (b *Batcher) Pending() int {
|
||||
n := 0
|
||||
for _, batch := range b.open {
|
||||
n += len(batch.Items)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func sortBatches(bs []*Batch) {
|
||||
sort.Slice(bs, func(i, j int) bool { return bs[i].Subject < bs[j].Subject })
|
||||
}
|
||||
|
||||
// NDJSON renders the batch as newline-delimited JSON (one raw event per line),
|
||||
// matching the raw archive format the logging stack expects.
|
||||
func (b *Batch) NDJSON() []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.Grow(int(b.RawBytes) + len(b.Items))
|
||||
for _, it := range b.Items {
|
||||
buf.Write(bytes.TrimRight(it.Raw, "\n"))
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// Summary is the index-relevant projection of a batch.
|
||||
type Summary struct {
|
||||
Hosts []string
|
||||
MinTS time.Time
|
||||
MaxTS time.Time
|
||||
EventCount int
|
||||
HasTS bool
|
||||
}
|
||||
|
||||
// Summarize computes hosts (unique, sorted) and the timestamp range. fallback is
|
||||
// used for events whose payload lacked a parseable timestamp (ingest time).
|
||||
func (b *Batch) Summarize(fallback time.Time) Summary {
|
||||
s := Summary{EventCount: len(b.Items)}
|
||||
hostSet := map[string]struct{}{}
|
||||
for _, it := range b.Items {
|
||||
if it.Host != "" {
|
||||
hostSet[it.Host] = struct{}{}
|
||||
}
|
||||
ts := it.Timestamp
|
||||
if !it.HasTS {
|
||||
ts = fallback
|
||||
} else {
|
||||
s.HasTS = true
|
||||
}
|
||||
if s.MinTS.IsZero() || ts.Before(s.MinTS) {
|
||||
s.MinTS = ts
|
||||
}
|
||||
if s.MaxTS.IsZero() || ts.After(s.MaxTS) {
|
||||
s.MaxTS = ts
|
||||
}
|
||||
}
|
||||
if s.MinTS.IsZero() {
|
||||
s.MinTS = fallback
|
||||
}
|
||||
if s.MaxTS.IsZero() {
|
||||
s.MaxTS = fallback
|
||||
}
|
||||
for h := range hostSet {
|
||||
s.Hosts = append(s.Hosts, h)
|
||||
}
|
||||
sort.Strings(s.Hosts)
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package batcher
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func item(subject, host string, ts time.Time, hasTS bool, raw string) Item {
|
||||
return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)}
|
||||
}
|
||||
|
||||
func TestFullByEvents(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 3})
|
||||
if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil {
|
||||
t.Fatalf("should not be full at 1")
|
||||
}
|
||||
if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil {
|
||||
t.Fatalf("should not be full at 2")
|
||||
}
|
||||
full := b.Add(item("s", "h", time.Now(), true, "c"))
|
||||
if full == nil {
|
||||
t.Fatalf("should be full at 3")
|
||||
}
|
||||
if len(full.Items) != 3 {
|
||||
t.Errorf("full batch has %d items", len(full.Items))
|
||||
}
|
||||
// After a full flush the subject batch is reset.
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("pending after flush = %d, want 0", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFullByBytes(t *testing.T) {
|
||||
b := New(Limits{MaxBytes: 10})
|
||||
if b.Add(item("s", "h", time.Now(), true, "12345")) != nil {
|
||||
t.Fatalf("5 bytes should not fill")
|
||||
}
|
||||
full := b.Add(item("s", "h", time.Now(), true, "67890"))
|
||||
if full == nil {
|
||||
t.Fatalf("10 bytes should fill")
|
||||
}
|
||||
if full.RawBytes != 10 {
|
||||
t.Errorf("RawBytes = %d", full.RawBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateSubjects(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 2})
|
||||
b.Add(item("a", "h", time.Now(), true, "x"))
|
||||
b.Add(item("b", "h", time.Now(), true, "y"))
|
||||
if b.Pending() != 2 {
|
||||
t.Errorf("pending = %d, want 2 across subjects", b.Pending())
|
||||
}
|
||||
full := b.Add(item("a", "h", time.Now(), true, "z"))
|
||||
if full == nil || full.Subject != "a" {
|
||||
t.Fatalf("subject a should flush independently")
|
||||
}
|
||||
if b.Pending() != 1 {
|
||||
t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueByAge(t *testing.T) {
|
||||
base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
b := New(Limits{MaxAge: time.Minute})
|
||||
b.nowFn = func() time.Time { return base }
|
||||
b.Add(item("s", "h", base, true, "x"))
|
||||
|
||||
if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 {
|
||||
t.Fatalf("not due at 30s")
|
||||
}
|
||||
due := b.DueByAge(base.Add(90 * time.Second))
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("should be due at 90s, got %d", len(due))
|
||||
}
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("due batch not removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDrain(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 100})
|
||||
b.Add(item("a", "h", time.Now(), true, "x"))
|
||||
b.Add(item("b", "h", time.Now(), true, "y"))
|
||||
all := b.Drain()
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("drain returned %d, want 2", len(all))
|
||||
}
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("pending after drain = %d", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNDJSON(t *testing.T) {
|
||||
b := &Batch{Subject: "s"}
|
||||
b.Items = []Item{
|
||||
{Raw: []byte(`{"a":1}`)},
|
||||
{Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added
|
||||
}
|
||||
got := string(b.NDJSON())
|
||||
want := "{\"a\":1}\n{\"b\":2}\n"
|
||||
if got != want {
|
||||
t.Errorf("NDJSON = %q, want %q", got, want)
|
||||
}
|
||||
if strings.Count(got, "\n") != 2 {
|
||||
t.Errorf("expected exactly 2 newlines")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize(t *testing.T) {
|
||||
t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
|
||||
t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
|
||||
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
b := &Batch{Subject: "s"}
|
||||
b.Items = []Item{
|
||||
item("s", "host-b", t2, true, "x"),
|
||||
item("s", "host-a", t1, true, "y"),
|
||||
item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host
|
||||
item("s", "host-a", t1, true, "w"), // dup host
|
||||
}
|
||||
s := b.Summarize(fallback)
|
||||
if s.EventCount != 4 {
|
||||
t.Errorf("EventCount = %d", s.EventCount)
|
||||
}
|
||||
if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" {
|
||||
t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts)
|
||||
}
|
||||
if !s.MinTS.Equal(t1) {
|
||||
t.Errorf("MinTS = %v, want %v", s.MinTS, t1)
|
||||
}
|
||||
// max should be the fallback (9:00) since event z used fallback which is latest
|
||||
if !s.MaxTS.Equal(fallback) {
|
||||
t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeAllFallback(t *testing.T) {
|
||||
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
b := &Batch{Items: []Item{{Raw: []byte("x")}}}
|
||||
s := b.Summarize(fallback)
|
||||
if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) {
|
||||
t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS)
|
||||
}
|
||||
if s.HasTS {
|
||||
t.Errorf("HasTS should be false")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/index"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// selectFlags are the shared object-selection flags for search and fetch.
|
||||
type selectFlags struct {
|
||||
subject string
|
||||
host string
|
||||
from string
|
||||
to string
|
||||
limit int
|
||||
}
|
||||
|
||||
func (s *selectFlags) bind(cmd *cobra.Command) {
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&s.subject, "subject", "", "NATS-style subject glob (e.g. 'logs.vm.*' or 'logs.k8s.vault.>')")
|
||||
f.StringVar(&s.host, "host", "", "source host to match (exact, or a glob with '*')")
|
||||
f.StringVar(&s.from, "from", "", "start of time window (RFC3339, 'YYYY-MM-DD', or relative like '-24h')")
|
||||
f.StringVar(&s.to, "to", "", "end of time window (RFC3339, 'YYYY-MM-DD', or relative like '-1h')")
|
||||
f.IntVar(&s.limit, "limit", 100, "max objects to return (0 = no limit)")
|
||||
}
|
||||
|
||||
// query builds an index.SearchQuery from the flags.
|
||||
func (s *selectFlags) query(now time.Time) (index.SearchQuery, error) {
|
||||
q := index.SearchQuery{Subject: s.subject, Host: s.host, Limit: s.limit}
|
||||
if s.from != "" {
|
||||
t, err := parseTimeArg(s.from, now)
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("--from: %w", err)
|
||||
}
|
||||
q.From = t
|
||||
}
|
||||
if s.to != "" {
|
||||
t, err := parseTimeArg(s.to, now)
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("--to: %w", err)
|
||||
}
|
||||
q.To = t
|
||||
}
|
||||
if !q.From.IsZero() && !q.To.IsZero() && q.To.Before(q.From) {
|
||||
return q, fmt.Errorf("--to (%s) is before --from (%s)", q.To, q.From)
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// parseTimeArg accepts RFC3339[/Nano], "YYYY-MM-DD", "YYYY-MM-DDTHH:MM:SS", or a
|
||||
// signed Go duration relative to now (e.g. "-24h", "30m").
|
||||
func parseTimeArg(s string, now time.Time) (time.Time, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t.UTC(), nil
|
||||
}
|
||||
}
|
||||
if d, err := time.ParseDuration(s); err == nil {
|
||||
return now.Add(d).UTC(), nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unrecognized time %q (use RFC3339, YYYY-MM-DD, or a duration like -24h)", s)
|
||||
}
|
||||
|
||||
func newIndexClient(cmd *cobra.Command) (index.Index, error) {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !cfg.Index.Enabled {
|
||||
return nil, fmt.Errorf("index is disabled in config; search/fetch-by-query require the ClickHouse index")
|
||||
}
|
||||
return index.NewClickHouse(cmd.Context(), indexConfig(cfg.Index))
|
||||
}
|
||||
|
||||
func humanBytes(n uint64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%dB", n)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for x := n / unit; x >= unit; x /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/crypto"
|
||||
"git.unkin.net/unkin/logarchiver/internal/s3store"
|
||||
"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newFetchCmd() *cobra.Command {
|
||||
var sel selectFlags
|
||||
var output string
|
||||
cmd := &cobra.Command{
|
||||
Use: "fetch [object-key ...]",
|
||||
Short: "Download, decrypt and decompress archived objects to NDJSON",
|
||||
Long: `fetch retrieves archived objects, decrypts them via the Vault GPG engine
|
||||
(the engine decrypts only the tiny wrapped data key; the bulk is streamed and
|
||||
decrypted locally), decompresses the zstd bulk, and emits the original NDJSON.
|
||||
|
||||
Objects are selected either by object key arguments, or by the same
|
||||
--subject/--host/--from/--to query used by 'search'. When --host/--from/--to are
|
||||
given they ALSO re-filter the emitted events to just the matching lines.`,
|
||||
Example: ` logarchiver search --subject 'logs.k8s.vault.>' --from -1h
|
||||
logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o -
|
||||
logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
q, err := sel.query(now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Resolve object keys: explicit args, else via index search.
|
||||
keys := args
|
||||
if len(keys) == 0 {
|
||||
if !cfg.Index.Enabled {
|
||||
return fmt.Errorf("no object keys given and index is disabled")
|
||||
}
|
||||
idx, err := newIndexClient(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = idx.Close() }()
|
||||
results, err := idx.Search(cmd.Context(), q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range results {
|
||||
keys = append(keys, r.ObjectKey)
|
||||
}
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "no matching objects")
|
||||
return nil
|
||||
}
|
||||
|
||||
store, err := s3store.New(cmd.Context(), s3store.Config{
|
||||
Endpoint: cfg.S3.Endpoint,
|
||||
Bucket: cfg.S3.Bucket,
|
||||
Region: cfg.S3.Region,
|
||||
PathStyle: cfg.S3.PathStyle,
|
||||
CAFile: cfg.S3.CAFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 init: %w", err)
|
||||
}
|
||||
// Operator decrypt path: force token auth (ambient VAULT_TOKEN /
|
||||
// ~/.vault-token), like passv, regardless of the service auth_method.
|
||||
vcfg := vaultConfig(cfg.Crypto.Vault)
|
||||
vcfg.AuthMethod = "token"
|
||||
vc, err := vaultgpg.New(cmd.Context(), vcfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("vault init: %w", err)
|
||||
}
|
||||
|
||||
filter := newLineFilter(sel.host, q.From, q.To)
|
||||
|
||||
var failures int
|
||||
for _, key := range keys {
|
||||
if err := fetchOne(cmd.Context(), store, vc, key, output, filter); err != nil {
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "fetch %s: %v\n", key, err)
|
||||
failures++
|
||||
}
|
||||
}
|
||||
if failures > 0 {
|
||||
return fmt.Errorf("%d of %d objects failed", failures, len(keys))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
sel.bind(cmd)
|
||||
cmd.Flags().StringVarP(&output, "output", "o", "-",
|
||||
"output: '-' for stdout, or a directory to write one NDJSON file per object")
|
||||
return cmd
|
||||
}
|
||||
|
||||
// fetchOne downloads, decrypts and decompresses a single object, applying the
|
||||
// optional line filter, to stdout or a per-object file under a directory.
|
||||
func fetchOne(ctx context.Context, store s3store.ObjectStore, vc *vaultgpg.Client, key, output string, filter lineFilter) error {
|
||||
body, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
|
||||
// Buffer the (bounded) object so we can read the header for its key name
|
||||
// before decrypting.
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read object: %w", err)
|
||||
}
|
||||
hdr, _, err := crypto.ReadHeader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keyName := hdr.KeyName
|
||||
if keyName == "" {
|
||||
return fmt.Errorf("object header has no key_name")
|
||||
}
|
||||
unwrap := func(wrapped []byte) ([]byte, error) {
|
||||
return vc.Decrypt(ctx, keyName, wrapped)
|
||||
}
|
||||
|
||||
var dst io.Writer
|
||||
var closer io.Closer
|
||||
if output == "-" || output == "" {
|
||||
dst = os.Stdout
|
||||
} else {
|
||||
if err := os.MkdirAll(output, 0o755); err != nil {
|
||||
return fmt.Errorf("create output dir: %w", err)
|
||||
}
|
||||
outPath := filepath.Join(output, sanitizeKey(key))
|
||||
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create output subdir: %w", err)
|
||||
}
|
||||
f, err := os.Create(outPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create output file: %w", err)
|
||||
}
|
||||
dst = f
|
||||
closer = f
|
||||
}
|
||||
|
||||
fw := newFilterWriter(dst, filter)
|
||||
if err := crypto.Open(bytes.NewReader(data), fw, unwrap); err != nil {
|
||||
if closer != nil {
|
||||
_ = closer.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := fw.Flush(); err != nil {
|
||||
if closer != nil {
|
||||
_ = closer.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
if closer != nil {
|
||||
return closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sanitizeKey turns an object key into a safe relative output filename, dropping
|
||||
// the .larc container suffix in favor of a plain .ndjson.
|
||||
func sanitizeKey(key string) string {
|
||||
name := strings.TrimSuffix(key, ".larc")
|
||||
if !strings.HasSuffix(name, ".ndjson") && !strings.HasSuffix(name, ".ndjson.zst") {
|
||||
name += ".ndjson"
|
||||
}
|
||||
name = strings.TrimSuffix(name, ".zst")
|
||||
return filepath.Clean("/" + name)[1:]
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/event"
|
||||
)
|
||||
|
||||
// lineFilter is a predicate over a single NDJSON event line.
|
||||
type lineFilter func(raw []byte) bool
|
||||
|
||||
// newLineFilter builds a predicate from optional host/time constraints. A nil
|
||||
// filter (all constraints empty) means "pass everything".
|
||||
func newLineFilter(host string, from, to time.Time) lineFilter {
|
||||
if host == "" && from.IsZero() && to.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return func(raw []byte) bool {
|
||||
meta := event.Extract(raw)
|
||||
if host != "" && !globMatch(host, meta.Host) {
|
||||
return false
|
||||
}
|
||||
if !from.IsZero() || !to.IsZero() {
|
||||
// Events without a parseable timestamp are kept (we cannot exclude
|
||||
// them on time grounds without dropping data).
|
||||
if meta.Ok {
|
||||
if !from.IsZero() && meta.Timestamp.Before(from) {
|
||||
return false
|
||||
}
|
||||
if !to.IsZero() && meta.Timestamp.After(to) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// filterWriter forwards only complete NDJSON lines that satisfy filter. It
|
||||
// buffers a trailing partial line across Writes so streaming decryption can feed
|
||||
// it arbitrary chunks. Flush must be called at end to emit any final unterminated
|
||||
// line. A nil filter forwards bytes verbatim.
|
||||
type filterWriter struct {
|
||||
dst io.Writer
|
||||
filter lineFilter
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func newFilterWriter(dst io.Writer, filter lineFilter) *filterWriter {
|
||||
return &filterWriter{dst: dst, filter: filter}
|
||||
}
|
||||
|
||||
func (w *filterWriter) Write(p []byte) (int, error) {
|
||||
if w.filter == nil {
|
||||
return w.dst.Write(p)
|
||||
}
|
||||
w.buf.Write(p)
|
||||
for {
|
||||
data := w.buf.Bytes()
|
||||
i := bytes.IndexByte(data, '\n')
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
line := data[:i]
|
||||
if len(bytes.TrimSpace(line)) > 0 && w.filter(line) {
|
||||
if _, err := w.dst.Write(line); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := w.dst.Write([]byte{'\n'}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
w.buf.Next(i + 1)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// Flush emits a trailing line that had no terminating newline.
|
||||
func (w *filterWriter) Flush() error {
|
||||
if w.filter == nil {
|
||||
return nil
|
||||
}
|
||||
line := bytes.TrimRight(w.buf.Bytes(), "\n")
|
||||
w.buf.Reset()
|
||||
if len(bytes.TrimSpace(line)) > 0 && w.filter(line) {
|
||||
if _, err := w.dst.Write(line); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.dst.Write([]byte{'\n'}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// globMatch matches pattern against s where '*' matches any run of characters.
|
||||
// With no '*', it is an exact match.
|
||||
func globMatch(pattern, s string) bool {
|
||||
if !strings.Contains(pattern, "*") {
|
||||
return pattern == s
|
||||
}
|
||||
parts := strings.Split(pattern, "*")
|
||||
// Anchor first part.
|
||||
if !strings.HasPrefix(s, parts[0]) {
|
||||
return false
|
||||
}
|
||||
s = s[len(parts[0]):]
|
||||
for _, part := range parts[1 : len(parts)-1] {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(s, part)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
s = s[idx+len(part):]
|
||||
}
|
||||
// Anchor last part.
|
||||
return strings.HasSuffix(s, parts[len(parts)-1])
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGlobMatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
pattern, s string
|
||||
want bool
|
||||
}{
|
||||
{"node-1", "node-1", true},
|
||||
{"node-1", "node-2", false},
|
||||
{"db-*", "db-1", true},
|
||||
{"db-*", "web-1", false},
|
||||
{"*-1", "node-1", true},
|
||||
{"*vault*", "logs-vault-audit", true},
|
||||
{"*vault*", "logs-web", false},
|
||||
{"a*b*c", "axxbyyc", true},
|
||||
{"a*b*c", "axxc", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := globMatch(c.pattern, c.s); got != c.want {
|
||||
t.Errorf("globMatch(%q,%q) = %v, want %v", c.pattern, c.s, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterWriterPassAll(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
fw := newFilterWriter(&out, nil) // nil filter = passthrough
|
||||
_, _ = fw.Write([]byte("line1\nline2\n"))
|
||||
_ = fw.Flush()
|
||||
if out.String() != "line1\nline2\n" {
|
||||
t.Errorf("passthrough altered data: %q", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterWriterHostFilterAcrossChunks(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
filter := newLineFilter("node-1", time.Time{}, time.Time{})
|
||||
fw := newFilterWriter(&out, filter)
|
||||
// Feed a line split across two Writes to exercise buffering.
|
||||
_, _ = fw.Write([]byte(`{"host":"node-1","m":"keep"}` + "\n" + `{"host":"node`))
|
||||
_, _ = fw.Write([]byte(`-2","m":"drop"}` + "\n" + `{"host":"node-1","m":"keep2"}` + "\n"))
|
||||
_ = fw.Flush()
|
||||
|
||||
got := out.String()
|
||||
if want := `{"host":"node-1","m":"keep"}` + "\n" + `{"host":"node-1","m":"keep2"}` + "\n"; got != want {
|
||||
t.Errorf("filtered output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterWriterTimeWindow(t *testing.T) {
|
||||
from := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
|
||||
to := time.Date(2026, 7, 27, 2, 0, 0, 0, time.UTC)
|
||||
var out bytes.Buffer
|
||||
fw := newFilterWriter(&out, newLineFilter("", from, to))
|
||||
lines := `{"host":"h","timestamp":"2026-07-27T00:30:00Z","m":"before"}` + "\n" +
|
||||
`{"host":"h","timestamp":"2026-07-27T01:30:00Z","m":"in"}` + "\n" +
|
||||
`{"host":"h","timestamp":"2026-07-27T03:00:00Z","m":"after"}` + "\n" +
|
||||
`{"host":"h","m":"no-ts-kept"}` + "\n"
|
||||
_, _ = fw.Write([]byte(lines))
|
||||
_ = fw.Flush()
|
||||
|
||||
got := out.String()
|
||||
if !bytes.Contains(out.Bytes(), []byte(`"in"`)) {
|
||||
t.Errorf("in-window line dropped: %q", got)
|
||||
}
|
||||
if bytes.Contains(out.Bytes(), []byte(`"before"`)) || bytes.Contains(out.Bytes(), []byte(`"after"`)) {
|
||||
t.Errorf("out-of-window line kept: %q", got)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte(`"no-ts-kept"`)) {
|
||||
t.Errorf("event without timestamp should be kept: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimeArg(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
rfc, err := parseTimeArg("2026-07-27T01:00:00Z", now)
|
||||
if err != nil || !rfc.Equal(time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("RFC3339 parse: %v %v", rfc, err)
|
||||
}
|
||||
d, err := parseTimeArg("2026-07-27", now)
|
||||
if err != nil || !d.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("date parse: %v %v", d, err)
|
||||
}
|
||||
rel, err := parseTimeArg("-24h", now)
|
||||
if err != nil || !rel.Equal(now.Add(-24*time.Hour)) {
|
||||
t.Errorf("relative parse: %v %v", rel, err)
|
||||
}
|
||||
if _, err := parseTimeArg("nonsense", now); err == nil {
|
||||
t.Errorf("expected error for nonsense time")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectFlagsQueryOrdering(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
s := &selectFlags{from: "-1h", to: "-2h"} // to before from
|
||||
if _, err := s.query(now); err == nil {
|
||||
t.Errorf("expected error when --to before --from")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeKey(t *testing.T) {
|
||||
got := sanitizeKey("archive/logs.vm.db-1/2026/07/27/20260727T101500Z-abcd.ndjson.zst.larc")
|
||||
if got != "archive/logs.vm.db-1/2026/07/27/20260727T101500Z-abcd.ndjson" {
|
||||
t.Errorf("sanitizeKey = %q", got)
|
||||
}
|
||||
// Path traversal is neutralized.
|
||||
if bad := sanitizeKey("../../etc/passwd"); bad != "etc/passwd.ndjson" {
|
||||
t.Errorf("sanitizeKey traversal = %q", bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/index"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newInitSchemaCmd() *cobra.Command {
|
||||
var printOnly bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "init-schema",
|
||||
Short: "Create the ClickHouse archive-index database and table (idempotent)",
|
||||
Long: `init-schema creates the ClickHouse database and archive_index table.
|
||||
|
||||
In-cluster the argocd bootstrap Job owns schema creation (like the logging
|
||||
stack's clickhouse-schema PostSync hook); this command is for local/dev use and
|
||||
for emitting the DDL (--print) to embed in that Job.`,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if printOnly {
|
||||
out := cmd.OutOrStdout()
|
||||
_, _ = fmt.Fprintln(out, index.CreateDatabaseSQL(cfg.Index.Database)+";")
|
||||
_, _ = fmt.Fprintln(out, index.CreateTableSQL(cfg.Index.Database, cfg.Index.Table)+";")
|
||||
return nil
|
||||
}
|
||||
ch, err := index.NewClickHouse(cmd.Context(), indexConfig(cfg.Index))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = ch.Close() }()
|
||||
if err := ch.InitSchema(cmd.Context()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "schema ready: %s.%s\n", cfg.Index.Database, cfg.Index.Table)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&printOnly, "print", false, "print the DDL instead of executing it")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Package cli implements the logarchiver command tree (service + operator CLI)
|
||||
// using cobra, which also provides the `completion` subcommand the estate's
|
||||
// nfpm packaging installs.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// version is set at build time via -ldflags "-X ...cli.version=...".
|
||||
var version = "dev"
|
||||
|
||||
// SetVersion lets main inject the linker-provided version string.
|
||||
func SetVersion(v string) {
|
||||
if v != "" {
|
||||
version = v
|
||||
}
|
||||
}
|
||||
|
||||
var configPath string
|
||||
|
||||
// NewRootCmd builds the root command.
|
||||
func NewRootCmd() *cobra.Command {
|
||||
root := &cobra.Command{
|
||||
Use: "logarchiver",
|
||||
Short: "Archive NATS JetStream logs to S3 (zstd + OpenPGP) and search/retrieve them",
|
||||
Long: `logarchiver archives raw logs from the centralized logging JetStream stream
|
||||
to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, and provides a
|
||||
CLI to search the index and retrieve/decrypt archived logs.`,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
root.PersistentFlags().StringVarP(&configPath, "config", "c", os.Getenv("LOGARCHIVER_CONFIG"),
|
||||
"path to config file (env LOGARCHIVER_CONFIG)")
|
||||
|
||||
root.AddCommand(
|
||||
newRunCmd(),
|
||||
newSearchCmd(),
|
||||
newFetchCmd(),
|
||||
newInitSchemaCmd(),
|
||||
newVersionCmd(),
|
||||
)
|
||||
return root
|
||||
}
|
||||
|
||||
// Execute runs the root command.
|
||||
func Execute() int {
|
||||
if err := NewRootCmd().Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func newVersionCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print the version",
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_, _ = fmt.Fprintln(cmd.OutOrStdout(), version)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// loadConfig loads config from the --config path (or defaults+env).
|
||||
func loadConfig() (config.Config, error) {
|
||||
return config.Load(configPath)
|
||||
}
|
||||
|
||||
// newLogger builds a slog logger from config.
|
||||
func newLogger(cfg config.LogConfig) *slog.Logger {
|
||||
level := slog.LevelInfo
|
||||
switch strings.ToLower(cfg.Level) {
|
||||
case "debug":
|
||||
level = slog.LevelDebug
|
||||
case "warn":
|
||||
level = slog.LevelWarn
|
||||
case "error":
|
||||
level = slog.LevelError
|
||||
}
|
||||
opts := &slog.HandlerOptions{Level: level}
|
||||
var h slog.Handler
|
||||
if strings.ToLower(cfg.Format) == "text" {
|
||||
h = slog.NewTextHandler(os.Stderr, opts)
|
||||
} else {
|
||||
h = slog.NewJSONHandler(os.Stderr, opts)
|
||||
}
|
||||
return slog.New(h)
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/archiver"
|
||||
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
||||
"git.unkin.net/unkin/logarchiver/internal/config"
|
||||
"git.unkin.net/unkin/logarchiver/internal/consumer"
|
||||
"git.unkin.net/unkin/logarchiver/internal/index"
|
||||
"git.unkin.net/unkin/logarchiver/internal/metrics"
|
||||
"git.unkin.net/unkin/logarchiver/internal/s3store"
|
||||
"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newRunCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "Run the archiver service (JetStream consumer -> S3 + index)",
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runService(cmd.Context(), cfg)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// persistAdapter bridges *archiver.Archiver to consumer.Persister (different
|
||||
// StoreResult types across package boundaries).
|
||||
type persistAdapter struct{ a *archiver.Archiver }
|
||||
|
||||
func (p persistAdapter) Store(ctx context.Context, b *batcher.Batch) (consumer.StoreResult, error) {
|
||||
res, err := p.a.Store(ctx, b)
|
||||
return consumer.StoreResult{
|
||||
ObjectKey: res.ObjectKey,
|
||||
Events: res.Events,
|
||||
RawBytes: res.RawBytes,
|
||||
StoredBytes: res.StoredBytes,
|
||||
}, err
|
||||
}
|
||||
|
||||
func runService(parent context.Context, cfg config.Config) error {
|
||||
log := newLogger(cfg.Log)
|
||||
slog.SetDefault(log)
|
||||
|
||||
ctx, stop := signal.NotifyContext(parent, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
// Metrics.
|
||||
var met *metrics.Metrics
|
||||
var metricsSrv *http.Server
|
||||
if cfg.Metrics.Enabled {
|
||||
reg := prometheus.NewRegistry()
|
||||
met = metrics.New(reg)
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
metricsSrv = &http.Server{Addr: cfg.Metrics.Address, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
go func() {
|
||||
log.Info("metrics listening", "addr", cfg.Metrics.Address)
|
||||
if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Error("metrics server failed", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Vault client only when we source the public key from Vault; the service
|
||||
// never needs Vault otherwise (file-mounted pubkey is the default).
|
||||
var vc *vaultgpg.Client
|
||||
if cfg.Crypto.Source == config.PubkeyVault {
|
||||
var err error
|
||||
vc, err = vaultgpg.New(ctx, vaultConfig(cfg.Crypto.Vault))
|
||||
if err != nil {
|
||||
return fmt.Errorf("vault init: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Public key provider.
|
||||
loader, err := archiver.PubkeyLoaderFromConfig(cfg.Crypto, vc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pubkeys, err := archiver.NewPubkeyProvider(ctx, loader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load public key: %w", err)
|
||||
}
|
||||
log.Info("public key loaded",
|
||||
"source", cfg.Crypto.Source, "key_name", cfg.Crypto.KeyName,
|
||||
"fingerprint", pubkeys.Current().Fingerprint)
|
||||
go refreshPubkey(ctx, log, pubkeys, cfg.Crypto.RefreshInterval)
|
||||
|
||||
// S3.
|
||||
store, err := s3store.New(ctx, s3store.Config{
|
||||
Endpoint: cfg.S3.Endpoint,
|
||||
Bucket: cfg.S3.Bucket,
|
||||
Region: cfg.S3.Region,
|
||||
PathStyle: cfg.S3.PathStyle,
|
||||
CAFile: cfg.S3.CAFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 init: %w", err)
|
||||
}
|
||||
|
||||
// Index.
|
||||
var idx index.Index
|
||||
if cfg.Index.Enabled {
|
||||
ch, err := index.NewClickHouse(ctx, indexConfig(cfg.Index))
|
||||
if err != nil {
|
||||
return fmt.Errorf("clickhouse init: %w", err)
|
||||
}
|
||||
defer func() { _ = ch.Close() }()
|
||||
idx = ch
|
||||
}
|
||||
|
||||
keys, err := archiver.NewKeyBuilder(cfg.S3.KeyPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
arch, err := archiver.New(archiver.Options{
|
||||
Keys: keys,
|
||||
Pubkeys: pubkeys,
|
||||
Store: store,
|
||||
Index: idx,
|
||||
KeyName: cfg.Crypto.KeyName,
|
||||
FrameSize: cfg.Crypto.FrameSize,
|
||||
Metrics: met,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// NATS + consumer.
|
||||
nc, js, err := consumer.Connect(cfg.NATS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nc.Close()
|
||||
cons, err := consumer.EnsureConsumer(ctx, js, cfg.NATS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("consumer bound",
|
||||
"stream", cfg.NATS.Stream, "durable", cfg.NATS.Durable, "subjects", cfg.NATS.Subjects)
|
||||
|
||||
bat := batcher.New(batcher.Limits{
|
||||
MaxBytes: cfg.Batch.MaxBytes,
|
||||
MaxEvents: cfg.Batch.MaxEvents,
|
||||
MaxAge: cfg.Batch.MaxAge,
|
||||
})
|
||||
|
||||
var runnerMetrics consumer.Metrics
|
||||
if met != nil {
|
||||
runnerMetrics = met
|
||||
}
|
||||
runner := consumer.NewRunner(consumer.Options{
|
||||
Consumer: cons,
|
||||
Batcher: bat,
|
||||
Persister: persistAdapter{a: arch},
|
||||
Logger: log,
|
||||
Metrics: runnerMetrics,
|
||||
FetchBatch: cfg.NATS.FetchBatch,
|
||||
PollWait: pollWait(cfg.Batch.MaxAge),
|
||||
})
|
||||
|
||||
runErr := runner.Run(ctx)
|
||||
|
||||
if metricsSrv != nil {
|
||||
shCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = metricsSrv.Shutdown(shCtx)
|
||||
cancel()
|
||||
}
|
||||
if runErr != nil && !errors.Is(runErr, context.Canceled) {
|
||||
return runErr
|
||||
}
|
||||
log.Info("shutdown complete")
|
||||
return nil
|
||||
}
|
||||
|
||||
// pollWait picks a fetch/age-check interval that is a fraction of MaxAge so
|
||||
// aged batches flush promptly, clamped to a sane range.
|
||||
func pollWait(maxAge time.Duration) time.Duration {
|
||||
if maxAge <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
w := maxAge / 10
|
||||
if w < time.Second {
|
||||
w = time.Second
|
||||
}
|
||||
if w > 10*time.Second {
|
||||
w = 10 * time.Second
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func refreshPubkey(ctx context.Context, log *slog.Logger, p *archiver.PubkeyProvider, every time.Duration) {
|
||||
if every <= 0 {
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if err := p.Refresh(ctx); err != nil {
|
||||
log.Warn("pubkey refresh failed; keeping previous key", "err", err)
|
||||
continue
|
||||
}
|
||||
log.Debug("pubkey refreshed", "fingerprint", p.Current().Fingerprint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func vaultConfig(v config.VaultConfig) vaultgpg.Config {
|
||||
return vaultgpg.Config{
|
||||
Address: v.Address,
|
||||
Mount: v.Mount,
|
||||
AuthMethod: v.AuthMethod,
|
||||
K8sRole: v.K8sRole,
|
||||
K8sMount: v.K8sMount,
|
||||
K8sJWTPath: v.K8sJWTPath,
|
||||
CAFile: v.CAFile,
|
||||
}
|
||||
}
|
||||
|
||||
func indexConfig(i config.IndexConfig) index.Config {
|
||||
return index.Config{
|
||||
Address: i.Address,
|
||||
Database: i.Database,
|
||||
Table: i.Table,
|
||||
Username: i.Username,
|
||||
Password: i.Password,
|
||||
TLS: i.TLS,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newSearchCmd() *cobra.Command {
|
||||
var sel selectFlags
|
||||
var asJSON bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "search",
|
||||
Short: "Search the archive index for matching objects",
|
||||
Long: `search queries the ClickHouse archive index and lists the S3 objects whose
|
||||
subject/host/time-range match, with event counts and sizes. Use the object keys
|
||||
with 'logarchiver fetch' to retrieve and decrypt their contents.`,
|
||||
Example: ` logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h`,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
q, err := sel.query(time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
idx, err := newIndexClient(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = idx.Close() }()
|
||||
|
||||
results, err := idx.Search(cmd.Context(), q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if asJSON {
|
||||
enc := json.NewEncoder(out)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(results)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
_, _ = fmt.Fprintln(out, "no matching objects")
|
||||
return nil
|
||||
}
|
||||
tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
|
||||
_, _ = fmt.Fprintln(tw, "OBJECT_KEY\tSUBJECT\tHOSTS\tMIN_TS\tMAX_TS\tEVENTS\tSTORED")
|
||||
var totalEvents, totalStored uint64
|
||||
for _, r := range results {
|
||||
_, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
r.ObjectKey, r.Subject, strings.Join(r.Hosts, ","),
|
||||
r.MinTS.UTC().Format(time.RFC3339), r.MaxTS.UTC().Format(time.RFC3339),
|
||||
r.EventCount, humanBytes(r.StoredBytes))
|
||||
totalEvents += r.EventCount
|
||||
totalStored += r.StoredBytes
|
||||
}
|
||||
_ = tw.Flush()
|
||||
_, _ = fmt.Fprintf(out, "\n%d objects, %d events, %s stored\n", len(results), totalEvents, humanBytes(totalStored))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
sel.bind(cmd)
|
||||
cmd.Flags().BoolVar(&asJSON, "json", false, "output results as JSON")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
// Package config defines logarchiver's configuration and loads it from a YAML
|
||||
// file with environment-variable overrides, so the same binary is
|
||||
// k8s-friendly (env/secret-driven) and laptop-friendly (a config file).
|
||||
//
|
||||
// Precedence: built-in defaults < YAML file < environment variables.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Config is the full logarchiver configuration.
|
||||
type Config struct {
|
||||
NATS NATSConfig `yaml:"nats"`
|
||||
Batch BatchConfig `yaml:"batch"`
|
||||
S3 S3Config `yaml:"s3"`
|
||||
Crypto CryptoConfig `yaml:"crypto"`
|
||||
Index IndexConfig `yaml:"index"`
|
||||
Metrics MetricsConfig `yaml:"metrics"`
|
||||
Log LogConfig `yaml:"log"`
|
||||
}
|
||||
|
||||
// NATSConfig configures the JetStream pull consumer that logarchiver binds. It
|
||||
// mirrors the logging stack's `LOGS` stream / `archiver` durable / `log-consumer`
|
||||
// user conventions (argocd-apps #296).
|
||||
type NATSConfig struct {
|
||||
URL string `yaml:"url"`
|
||||
Stream string `yaml:"stream"`
|
||||
Durable string `yaml:"durable"`
|
||||
Subjects []string `yaml:"subjects"`
|
||||
User string `yaml:"user"`
|
||||
// Password is the NATS user password. In-cluster it comes from the
|
||||
// nats-auth secret via NATS_CONSUMER_PASSWORD (see PasswordEnv).
|
||||
Password string `yaml:"password"`
|
||||
// PasswordEnv names the env var holding the password when Password is empty.
|
||||
PasswordEnv string `yaml:"password_env"`
|
||||
// CAFile trusts a custom CA for TLS to NATS (usually unset; in-cluster is plaintext).
|
||||
CAFile string `yaml:"ca_file"`
|
||||
// FetchBatch is the max messages pulled per Fetch call.
|
||||
FetchBatch int `yaml:"fetch_batch"`
|
||||
// AckWait is the JetStream redelivery timeout; must exceed a worst-case
|
||||
// batch flush (compress+encrypt+S3 PUT+index write).
|
||||
AckWait time.Duration `yaml:"ack_wait"`
|
||||
}
|
||||
|
||||
// BatchConfig bounds a single archived object. A per-subject batch is flushed
|
||||
// when any bound is hit. Keep MaxBytes well under the crypto/engine ceiling so
|
||||
// even a whole-object decrypt path stays viable; the wrapped-DEK envelope means
|
||||
// object size is not limited by Vault, but smaller objects retrieve faster.
|
||||
type BatchConfig struct {
|
||||
MaxBytes int64 `yaml:"max_bytes"`
|
||||
MaxEvents int `yaml:"max_events"`
|
||||
MaxAge time.Duration `yaml:"max_age"`
|
||||
}
|
||||
|
||||
// S3Config targets the Ceph RGW bucket. Credentials are read from the standard
|
||||
// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars (cephrgw BucketAccess
|
||||
// secret logs-archive-s3), so they are intentionally absent here.
|
||||
type S3Config struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Region string `yaml:"region"`
|
||||
PathStyle bool `yaml:"path_style"`
|
||||
// KeyPrefix is a text/template with {{.Subject}} {{.Year}} {{.Month}} {{.Day}}.
|
||||
KeyPrefix string `yaml:"key_prefix"`
|
||||
// CAFile trusts the internal Vault-PKI CA for https://s3.ceph.unkin.net.
|
||||
CAFile string `yaml:"ca_file"`
|
||||
// EndpointEnv / BucketEnv let the cephrgw secret (S3_ENDPOINT / BUCKET_NAME)
|
||||
// override endpoint/bucket without a config edit.
|
||||
EndpointEnv string `yaml:"endpoint_env"`
|
||||
BucketEnv string `yaml:"bucket_env"`
|
||||
}
|
||||
|
||||
// PubkeySource selects where the OpenPGP public key is fetched from.
|
||||
type PubkeySource string
|
||||
|
||||
const (
|
||||
PubkeyVault PubkeySource = "vault" // read gpg/keys/<name> from the Vault GPG engine
|
||||
PubkeyFile PubkeySource = "file" // read an armored public key from a mounted file
|
||||
)
|
||||
|
||||
// CryptoConfig controls encryption. The service only ever needs the PUBLIC key;
|
||||
// decryption (CLI fetch) always goes through the Vault GPG engine.
|
||||
type CryptoConfig struct {
|
||||
KeyName string `yaml:"key_name"`
|
||||
Source PubkeySource `yaml:"pubkey_source"`
|
||||
// PubkeyFile is the armored public key path when Source==file.
|
||||
PubkeyFile string `yaml:"pubkey_file"`
|
||||
// RefreshInterval re-fetches the public key periodically (rotation aware).
|
||||
RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// FrameSize is the AES-GCM frame plaintext size in bytes (streaming decrypt).
|
||||
FrameSize int `yaml:"frame_size"`
|
||||
Vault VaultConfig `yaml:"vault"`
|
||||
}
|
||||
|
||||
// VaultConfig configures access to the Vault GPG secrets engine. For the
|
||||
// service (pubkey fetch) k8s auth is used in-cluster; the CLI relies on the
|
||||
// operator's ambient VAULT_TOKEN (~/.vault-token), like passv.
|
||||
type VaultConfig struct {
|
||||
Address string `yaml:"address"`
|
||||
// Mount is the GPG engine mount path (e.g. "gpg").
|
||||
Mount string `yaml:"mount"`
|
||||
// AuthMethod is "token" or "kubernetes".
|
||||
AuthMethod string `yaml:"auth_method"`
|
||||
// K8sRole / K8sMount / K8sJWTPath configure kubernetes auth.
|
||||
K8sRole string `yaml:"k8s_role"`
|
||||
K8sMount string `yaml:"k8s_mount"`
|
||||
K8sJWTPath string `yaml:"k8s_jwt_path"`
|
||||
CAFile string `yaml:"ca_file"`
|
||||
}
|
||||
|
||||
// IndexConfig targets the ClickHouse archive index. Credentials come from the
|
||||
// clickhouse-credentials secret via env by default.
|
||||
type IndexConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Address string `yaml:"address"` // host:port for the native protocol (9000)
|
||||
Database string `yaml:"database"`
|
||||
Table string `yaml:"table"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordEnv string `yaml:"password_env"`
|
||||
TLS bool `yaml:"tls"`
|
||||
}
|
||||
|
||||
// MetricsConfig configures the Prometheus /metrics listener.
|
||||
type MetricsConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Address string `yaml:"address"`
|
||||
}
|
||||
|
||||
// LogConfig configures structured logging.
|
||||
type LogConfig struct {
|
||||
Level string `yaml:"level"` // debug|info|warn|error
|
||||
Format string `yaml:"format"` // json|text
|
||||
}
|
||||
|
||||
// Default returns a Config pre-populated with the logging-stack conventions so
|
||||
// an in-cluster deployment needs only secrets (creds) supplied via env.
|
||||
func Default() Config {
|
||||
return Config{
|
||||
NATS: NATSConfig{
|
||||
URL: "nats://nats.logging.svc.cluster.local:4222",
|
||||
Stream: "LOGS",
|
||||
Durable: "archiver",
|
||||
Subjects: []string{"logs.k8s.vault.>"},
|
||||
User: "log-consumer",
|
||||
PasswordEnv: "NATS_CONSUMER_PASSWORD",
|
||||
FetchBatch: 512,
|
||||
AckWait: 2 * time.Minute,
|
||||
},
|
||||
Batch: BatchConfig{
|
||||
MaxBytes: 64 * 1024 * 1024, // 64 MiB raw NDJSON per object
|
||||
MaxEvents: 200000,
|
||||
MaxAge: 5 * time.Minute,
|
||||
},
|
||||
S3: S3Config{
|
||||
Endpoint: "https://s3.ceph.unkin.net",
|
||||
Bucket: "logs-archive",
|
||||
Region: "us-east-1",
|
||||
PathStyle: true,
|
||||
KeyPrefix: "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/",
|
||||
CAFile: "/etc/vault-ca/ca.crt",
|
||||
EndpointEnv: "S3_ENDPOINT",
|
||||
BucketEnv: "BUCKET_NAME",
|
||||
},
|
||||
Crypto: CryptoConfig{
|
||||
KeyName: "logarchive",
|
||||
Source: PubkeyFile,
|
||||
PubkeyFile: "/etc/logarchiver/pubkey.asc",
|
||||
RefreshInterval: time.Hour,
|
||||
FrameSize: 1 << 20, // 1 MiB frames
|
||||
Vault: VaultConfig{
|
||||
Mount: "gpg",
|
||||
AuthMethod: "kubernetes",
|
||||
K8sMount: "k8s/au/syd1",
|
||||
K8sRole: "default",
|
||||
K8sJWTPath: "/var/run/secrets/kubernetes.io/serviceaccount/token",
|
||||
},
|
||||
},
|
||||
Index: IndexConfig{
|
||||
Enabled: true,
|
||||
Address: "clickhouse-logs.logging.svc.cluster.local:9000",
|
||||
Database: "logs",
|
||||
Table: "archive_index",
|
||||
Username: "vector",
|
||||
PasswordEnv: "CLICKHOUSE_PASSWORD",
|
||||
TLS: false,
|
||||
},
|
||||
Metrics: MetricsConfig{Enabled: true, Address: ":9090"},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
}
|
||||
}
|
||||
|
||||
// Load reads defaults, overlays the YAML file at path (if non-empty), then
|
||||
// applies environment overrides, and validates the result.
|
||||
func Load(path string) (Config, error) {
|
||||
cfg := Default()
|
||||
if path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("parse config %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
cfg.applyEnv()
|
||||
cfg.resolveSecretEnvs()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// applyEnv overlays scalar overrides from the environment. Only the knobs an
|
||||
// operator commonly flips are wired; secrets are handled by resolveSecretEnvs.
|
||||
func (c *Config) applyEnv() {
|
||||
if v := os.Getenv("LOGARCHIVER_NATS_URL"); v != "" {
|
||||
c.NATS.URL = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_NATS_DURABLE"); v != "" {
|
||||
c.NATS.Durable = v
|
||||
}
|
||||
if v := os.Getenv("ARCHIVE_SUBJECTS"); v != "" {
|
||||
c.NATS.Subjects = splitFields(v)
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_S3_ENDPOINT"); v != "" {
|
||||
c.S3.Endpoint = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_S3_BUCKET"); v != "" {
|
||||
c.S3.Bucket = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_KEY_NAME"); v != "" {
|
||||
c.Crypto.KeyName = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_PUBKEY_SOURCE"); v != "" {
|
||||
c.Crypto.Source = PubkeySource(v)
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_PUBKEY_FILE"); v != "" {
|
||||
c.Crypto.PubkeyFile = v
|
||||
}
|
||||
if v := os.Getenv("VAULT_ADDR"); v != "" && c.Crypto.Vault.Address == "" {
|
||||
c.Crypto.Vault.Address = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_VAULT_MOUNT"); v != "" {
|
||||
c.Crypto.Vault.Mount = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_CLICKHOUSE_ADDR"); v != "" {
|
||||
c.Index.Address = v
|
||||
}
|
||||
if v := os.Getenv("CLICKHOUSE_USER"); v != "" {
|
||||
c.Index.Username = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_METRICS_ADDR"); v != "" {
|
||||
c.Metrics.Address = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_LOG_LEVEL"); v != "" {
|
||||
c.Log.Level = v
|
||||
}
|
||||
if v := os.Getenv("LOGARCHIVER_LOG_FORMAT"); v != "" {
|
||||
c.Log.Format = v
|
||||
}
|
||||
// Endpoint/bucket sourced from the cephrgw secret, if present.
|
||||
if c.S3.EndpointEnv != "" {
|
||||
if v := os.Getenv(c.S3.EndpointEnv); v != "" {
|
||||
c.S3.Endpoint = v
|
||||
}
|
||||
}
|
||||
if c.S3.BucketEnv != "" {
|
||||
if v := os.Getenv(c.S3.BucketEnv); v != "" {
|
||||
c.S3.Bucket = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resolveSecretEnvs pulls passwords from their named env vars when not set inline.
|
||||
func (c *Config) resolveSecretEnvs() {
|
||||
if c.NATS.Password == "" && c.NATS.PasswordEnv != "" {
|
||||
c.NATS.Password = os.Getenv(c.NATS.PasswordEnv)
|
||||
}
|
||||
if c.Index.Password == "" && c.Index.PasswordEnv != "" {
|
||||
c.Index.Password = os.Getenv(c.Index.PasswordEnv)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks required fields and coherence.
|
||||
func (c *Config) Validate() error {
|
||||
if c.NATS.URL == "" {
|
||||
return fmt.Errorf("nats.url is required")
|
||||
}
|
||||
if c.NATS.Stream == "" {
|
||||
return fmt.Errorf("nats.stream is required")
|
||||
}
|
||||
if c.NATS.Durable == "" {
|
||||
return fmt.Errorf("nats.durable is required")
|
||||
}
|
||||
if len(c.NATS.Subjects) == 0 {
|
||||
return fmt.Errorf("nats.subjects must list at least one filter subject")
|
||||
}
|
||||
if c.S3.Bucket == "" {
|
||||
return fmt.Errorf("s3.bucket is required")
|
||||
}
|
||||
if c.S3.Endpoint == "" {
|
||||
return fmt.Errorf("s3.endpoint is required")
|
||||
}
|
||||
if c.Crypto.KeyName == "" {
|
||||
return fmt.Errorf("crypto.key_name is required")
|
||||
}
|
||||
switch c.Crypto.Source {
|
||||
case PubkeyVault:
|
||||
if c.Crypto.Vault.Address == "" {
|
||||
return fmt.Errorf("crypto.vault.address is required when pubkey_source=vault")
|
||||
}
|
||||
if c.Crypto.Vault.Mount == "" {
|
||||
return fmt.Errorf("crypto.vault.mount is required when pubkey_source=vault")
|
||||
}
|
||||
case PubkeyFile:
|
||||
if c.Crypto.PubkeyFile == "" {
|
||||
return fmt.Errorf("crypto.pubkey_file is required when pubkey_source=file")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("crypto.pubkey_source must be 'vault' or 'file', got %q", c.Crypto.Source)
|
||||
}
|
||||
if c.Crypto.FrameSize <= 0 {
|
||||
return fmt.Errorf("crypto.frame_size must be positive")
|
||||
}
|
||||
if c.Batch.MaxBytes <= 0 && c.Batch.MaxEvents <= 0 && c.Batch.MaxAge <= 0 {
|
||||
return fmt.Errorf("batch must set at least one of max_bytes/max_events/max_age")
|
||||
}
|
||||
if c.Index.Enabled && c.Index.Address == "" {
|
||||
return fmt.Errorf("index.address is required when index.enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitFields(s string) []string {
|
||||
var out []string
|
||||
for _, f := range strings.Fields(s) {
|
||||
if f != "" {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseSize parses a byte size like "64Mi", "128MB", "1024". It is a helper for
|
||||
// CLI flags; the YAML fields are plain integers.
|
||||
func ParseSize(s string) (int64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, fmt.Errorf("empty size")
|
||||
}
|
||||
mult := int64(1)
|
||||
switch {
|
||||
case strings.HasSuffix(s, "Gi"):
|
||||
mult, s = 1<<30, strings.TrimSuffix(s, "Gi")
|
||||
case strings.HasSuffix(s, "Mi"):
|
||||
mult, s = 1<<20, strings.TrimSuffix(s, "Mi")
|
||||
case strings.HasSuffix(s, "Ki"):
|
||||
mult, s = 1<<10, strings.TrimSuffix(s, "Ki")
|
||||
case strings.HasSuffix(s, "GB"):
|
||||
mult, s = 1e9, strings.TrimSuffix(s, "GB")
|
||||
case strings.HasSuffix(s, "MB"):
|
||||
mult, s = 1e6, strings.TrimSuffix(s, "MB")
|
||||
case strings.HasSuffix(s, "KB"):
|
||||
mult, s = 1e3, strings.TrimSuffix(s, "KB")
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse size %q: %w", s, err)
|
||||
}
|
||||
return n * mult, nil
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDefaultIsValid(t *testing.T) {
|
||||
cfg := Default()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("default config should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadEnvOverrides(t *testing.T) {
|
||||
t.Setenv("ARCHIVE_SUBJECTS", "logs.vm.> logs.k8s.vault.>")
|
||||
t.Setenv("LOGARCHIVER_S3_BUCKET", "my-bucket")
|
||||
t.Setenv("LOGARCHIVER_NATS_DURABLE", "archiver-canary")
|
||||
t.Setenv("NATS_CONSUMER_PASSWORD", "s3cr3t")
|
||||
t.Setenv("S3_ENDPOINT", "https://rgw.internal")
|
||||
t.Setenv("BUCKET_NAME", "logs-archive-override")
|
||||
|
||||
cfg, err := Load("")
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if got, want := cfg.NATS.Subjects, []string{"logs.vm.>", "logs.k8s.vault.>"}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
|
||||
t.Errorf("subjects = %v, want %v", got, want)
|
||||
}
|
||||
if cfg.NATS.Durable != "archiver-canary" {
|
||||
t.Errorf("durable = %q", cfg.NATS.Durable)
|
||||
}
|
||||
if cfg.NATS.Password != "s3cr3t" {
|
||||
t.Errorf("password from env not resolved: %q", cfg.NATS.Password)
|
||||
}
|
||||
// BUCKET_NAME (secret) should win over LOGARCHIVER_S3_BUCKET default flow.
|
||||
if cfg.S3.Bucket != "logs-archive-override" {
|
||||
t.Errorf("bucket = %q, want cephrgw secret override", cfg.S3.Bucket)
|
||||
}
|
||||
if cfg.S3.Endpoint != "https://rgw.internal" {
|
||||
t.Errorf("endpoint = %q", cfg.S3.Endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateErrors(t *testing.T) {
|
||||
cases := map[string]func(*Config){
|
||||
"no subjects": func(c *Config) { c.NATS.Subjects = nil },
|
||||
"no bucket": func(c *Config) { c.S3.Bucket = "" },
|
||||
"no key name": func(c *Config) { c.Crypto.KeyName = "" },
|
||||
"bad source": func(c *Config) { c.Crypto.Source = "elsewhere" },
|
||||
"file no path": func(c *Config) { c.Crypto.Source = PubkeyFile; c.Crypto.PubkeyFile = "" },
|
||||
"vault no addr": func(c *Config) { c.Crypto.Source = PubkeyVault; c.Crypto.Vault.Address = "" },
|
||||
"zero frame": func(c *Config) { c.Crypto.FrameSize = 0 },
|
||||
"no batch bounds": func(c *Config) { c.Batch = BatchConfig{} },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
mutate(&cfg)
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Errorf("expected validation error for %q", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateVaultSourceOK(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Crypto.Source = PubkeyVault
|
||||
cfg.Crypto.Vault.Address = "https://vault.example:8200"
|
||||
cfg.Crypto.Vault.Mount = "gpg"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("vault source should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSize(t *testing.T) {
|
||||
cases := map[string]int64{
|
||||
"1024": 1024,
|
||||
"64Mi": 64 << 20,
|
||||
"2Gi": 2 << 30,
|
||||
"1Ki": 1024,
|
||||
"5MB": 5_000_000,
|
||||
" 10 ": 10,
|
||||
}
|
||||
for in, want := range cases {
|
||||
got, err := ParseSize(in)
|
||||
if err != nil {
|
||||
t.Errorf("ParseSize(%q): %v", in, err)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("ParseSize(%q) = %d, want %d", in, got, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseSize("bogus"); err == nil {
|
||||
t.Errorf("expected error for bogus size")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultBatchDurations(t *testing.T) {
|
||||
cfg := Default()
|
||||
if cfg.Batch.MaxAge != 5*time.Minute {
|
||||
t.Errorf("default MaxAge = %v", cfg.Batch.MaxAge)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/config"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Connect dials NATS as the configured user and returns the connection and a
|
||||
// JetStream context. Callers must Close the returned *nats.Conn.
|
||||
func Connect(cfg config.NATSConfig) (*nats.Conn, jetstream.JetStream, error) {
|
||||
opts := []nats.Option{
|
||||
nats.Name("logarchiver"),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.ReconnectWait(2 * time.Second),
|
||||
}
|
||||
if cfg.User != "" {
|
||||
opts = append(opts, nats.UserInfo(cfg.User, cfg.Password))
|
||||
}
|
||||
if cfg.CAFile != "" {
|
||||
pool := x509.NewCertPool()
|
||||
pem, err := os.ReadFile(cfg.CAFile)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read nats ca %s: %w", cfg.CAFile, err)
|
||||
}
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, nil, fmt.Errorf("no certs parsed from nats ca %s", cfg.CAFile)
|
||||
}
|
||||
opts = append(opts, nats.Secure(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}))
|
||||
}
|
||||
nc, err := nats.Connect(cfg.URL, opts...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("connect nats %s: %w", cfg.URL, err)
|
||||
}
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, nil, fmt.Errorf("jetstream context: %w", err)
|
||||
}
|
||||
return nc, js, nil
|
||||
}
|
||||
|
||||
// EnsureConsumer creates or updates the durable pull consumer on the stream with
|
||||
// the configured subject filters. Independent offsets and explicit acks give
|
||||
// logarchiver at-least-once delivery decoupled from the transform tier.
|
||||
func EnsureConsumer(ctx context.Context, js jetstream.JetStream, cfg config.NATSConfig) (jetstream.Consumer, error) {
|
||||
ackWait := cfg.AckWait
|
||||
if ackWait <= 0 {
|
||||
ackWait = 2 * time.Minute
|
||||
}
|
||||
consCfg := jetstream.ConsumerConfig{
|
||||
Durable: cfg.Durable,
|
||||
Name: cfg.Durable,
|
||||
AckPolicy: jetstream.AckExplicitPolicy,
|
||||
DeliverPolicy: jetstream.DeliverAllPolicy,
|
||||
AckWait: ackWait,
|
||||
MaxDeliver: -1,
|
||||
ReplayPolicy: jetstream.ReplayInstantPolicy,
|
||||
}
|
||||
switch len(cfg.Subjects) {
|
||||
case 0:
|
||||
return nil, fmt.Errorf("no subject filters configured")
|
||||
case 1:
|
||||
consCfg.FilterSubject = cfg.Subjects[0]
|
||||
default:
|
||||
consCfg.FilterSubjects = cfg.Subjects
|
||||
}
|
||||
cons, err := js.CreateOrUpdateConsumer(ctx, cfg.Stream, consCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ensure consumer %s on stream %s: %w", cfg.Durable, cfg.Stream, err)
|
||||
}
|
||||
return cons, nil
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package consumer
|
||||
|
||||
import "git.unkin.net/unkin/logarchiver/internal/event"
|
||||
|
||||
// extractMeta projects the host/timestamp fields from a raw event payload.
|
||||
func extractMeta(raw []byte) event.Meta {
|
||||
return event.Extract(raw)
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
// Package consumer binds the JetStream pull consumer and runs the archive loop.
|
||||
//
|
||||
// The core correctness property: a batch's messages are acknowledged ONLY after
|
||||
// the batch has been sealed, uploaded to S3, and indexed in ClickHouse. If any
|
||||
// of those fails the messages are Nak'd (with a backoff) and JetStream
|
||||
// redelivers them, so nothing is lost on a sink outage. This sink-conditional
|
||||
// acking is the main thing logarchiver does that a stock Vector NATS consumer
|
||||
// cannot.
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Persister stores a ready batch durably. On success the caller acks.
|
||||
type Persister interface {
|
||||
Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error)
|
||||
}
|
||||
|
||||
// StoreResult mirrors archiver.StoreResult (kept local to avoid an import cycle;
|
||||
// the archiver's result is adapted at the call site).
|
||||
type StoreResult struct {
|
||||
ObjectKey string
|
||||
Events int
|
||||
RawBytes int64
|
||||
StoredBytes int64
|
||||
}
|
||||
|
||||
// Metrics is the optional metrics surface for the loop.
|
||||
type Metrics interface {
|
||||
MessagesFetched(n int)
|
||||
Acked(n int)
|
||||
BatchFlushed(trigger string)
|
||||
SetPending(n int)
|
||||
}
|
||||
|
||||
// Runner drives the fetch → batch → persist → ack loop.
|
||||
type Runner struct {
|
||||
cons jetstream.Consumer
|
||||
batcher *batcher.Batcher
|
||||
persist Persister
|
||||
log *slog.Logger
|
||||
metrics Metrics
|
||||
fetchBatch int
|
||||
pollWait time.Duration
|
||||
nakBackoff time.Duration
|
||||
drainTO time.Duration
|
||||
nowFn func() time.Time
|
||||
}
|
||||
|
||||
// Options configures a Runner.
|
||||
type Options struct {
|
||||
Consumer jetstream.Consumer
|
||||
Batcher *batcher.Batcher
|
||||
Persister Persister
|
||||
Logger *slog.Logger
|
||||
Metrics Metrics
|
||||
FetchBatch int
|
||||
// PollWait bounds each Fetch and thus how often age-based flushes are checked.
|
||||
PollWait time.Duration
|
||||
// NakBackoff delays redelivery after a persist failure.
|
||||
NakBackoff time.Duration
|
||||
// DrainTimeout bounds the shutdown flush.
|
||||
DrainTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewRunner builds a Runner.
|
||||
func NewRunner(o Options) *Runner {
|
||||
fetch := o.FetchBatch
|
||||
if fetch <= 0 {
|
||||
fetch = 512
|
||||
}
|
||||
poll := o.PollWait
|
||||
if poll <= 0 {
|
||||
poll = time.Second
|
||||
}
|
||||
nak := o.NakBackoff
|
||||
if nak <= 0 {
|
||||
nak = 10 * time.Second
|
||||
}
|
||||
drain := o.DrainTimeout
|
||||
if drain <= 0 {
|
||||
drain = 30 * time.Second
|
||||
}
|
||||
log := o.Logger
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &Runner{
|
||||
cons: o.Consumer,
|
||||
batcher: o.Batcher,
|
||||
persist: o.Persister,
|
||||
log: log,
|
||||
metrics: o.Metrics,
|
||||
fetchBatch: fetch,
|
||||
pollWait: poll,
|
||||
nakBackoff: nak,
|
||||
drainTO: drain,
|
||||
nowFn: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// Run loops until ctx is cancelled, then drains open batches before returning.
|
||||
func (r *Runner) Run(ctx context.Context) error {
|
||||
r.log.Info("archive loop started",
|
||||
"fetch_batch", r.fetchBatch, "poll_wait", r.pollWait.String())
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return r.drain()
|
||||
}
|
||||
|
||||
// Age-based flush before fetching more.
|
||||
r.flushBatches(ctx, r.batcher.DueByAge(r.nowFn()), "age")
|
||||
|
||||
msgs, err := r.cons.Fetch(r.fetchBatch, jetstream.FetchMaxWait(r.pollWait))
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||
return r.drain()
|
||||
}
|
||||
r.log.Warn("fetch failed", "err", err)
|
||||
r.sleep(ctx, r.pollWait)
|
||||
continue
|
||||
}
|
||||
|
||||
n := 0
|
||||
for msg := range msgs.Messages() {
|
||||
n++
|
||||
r.route(ctx, msg)
|
||||
}
|
||||
if ferr := msgs.Error(); ferr != nil && !errors.Is(ferr, context.Canceled) {
|
||||
r.log.Warn("fetch iteration error", "err", ferr)
|
||||
}
|
||||
if r.metrics != nil {
|
||||
r.metrics.MessagesFetched(n)
|
||||
r.metrics.SetPending(r.batcher.Pending())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// route decodes a message and adds it to the batcher, flushing if the batch
|
||||
// becomes full.
|
||||
func (r *Runner) route(ctx context.Context, msg jetstream.Msg) {
|
||||
meta := extractMeta(msg.Data())
|
||||
full := r.batcher.Add(batcher.Item{
|
||||
Subject: msg.Subject(),
|
||||
Raw: msg.Data(),
|
||||
Host: meta.Host,
|
||||
Timestamp: meta.Timestamp,
|
||||
HasTS: meta.Ok,
|
||||
Ack: msg,
|
||||
})
|
||||
if full != nil {
|
||||
r.flush(ctx, full, "full")
|
||||
}
|
||||
}
|
||||
|
||||
// flushBatches flushes a slice of batches with the given trigger label.
|
||||
func (r *Runner) flushBatches(ctx context.Context, batches []*batcher.Batch, trigger string) {
|
||||
for _, b := range batches {
|
||||
r.flush(ctx, b, trigger)
|
||||
}
|
||||
}
|
||||
|
||||
// flush persists a batch and, only on success, acks its messages. On failure it
|
||||
// Naks with a backoff so JetStream redelivers.
|
||||
func (r *Runner) flush(ctx context.Context, b *batcher.Batch, trigger string) {
|
||||
if len(b.Items) == 0 {
|
||||
return
|
||||
}
|
||||
res, err := r.persist.Store(ctx, b)
|
||||
if err != nil {
|
||||
r.log.Error("persist failed; batch will be redelivered",
|
||||
"subject", b.Subject, "events", len(b.Items), "trigger", trigger, "err", err)
|
||||
r.nakAll(b)
|
||||
return
|
||||
}
|
||||
acked := r.ackAll(b)
|
||||
if r.metrics != nil {
|
||||
r.metrics.Acked(acked)
|
||||
r.metrics.BatchFlushed(trigger)
|
||||
}
|
||||
r.log.Info("object archived",
|
||||
"subject", b.Subject, "object_key", res.ObjectKey,
|
||||
"events", res.Events, "raw_bytes", res.RawBytes, "stored_bytes", res.StoredBytes,
|
||||
"trigger", trigger)
|
||||
}
|
||||
|
||||
func (r *Runner) ackAll(b *batcher.Batch) int {
|
||||
n := 0
|
||||
for _, it := range b.Items {
|
||||
if msg, ok := it.Ack.(jetstream.Msg); ok {
|
||||
if err := msg.Ack(); err != nil {
|
||||
r.log.Warn("ack failed", "subject", b.Subject, "err", err)
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (r *Runner) nakAll(b *batcher.Batch) {
|
||||
for _, it := range b.Items {
|
||||
if msg, ok := it.Ack.(jetstream.Msg); ok {
|
||||
_ = msg.NakWithDelay(r.nakBackoff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drain flushes all open batches during shutdown with a bounded timeout.
|
||||
func (r *Runner) drain() error {
|
||||
batches := r.batcher.Drain()
|
||||
if len(batches) == 0 {
|
||||
r.log.Info("archive loop stopped; nothing to drain")
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), r.drainTO)
|
||||
defer cancel()
|
||||
r.log.Info("draining open batches", "batches", len(batches))
|
||||
r.flushBatches(ctx, batches, "shutdown")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) sleep(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// fakeMsg is a minimal jetstream.Msg recording ack/nak calls.
|
||||
type fakeMsg struct {
|
||||
subject string
|
||||
data []byte
|
||||
mu sync.Mutex
|
||||
acked bool
|
||||
naked bool
|
||||
}
|
||||
|
||||
func (m *fakeMsg) Metadata() (*jetstream.MsgMetadata, error) { return &jetstream.MsgMetadata{}, nil }
|
||||
func (m *fakeMsg) Data() []byte { return m.data }
|
||||
func (m *fakeMsg) Headers() nats.Header { return nil }
|
||||
func (m *fakeMsg) Subject() string { return m.subject }
|
||||
func (m *fakeMsg) Reply() string { return "" }
|
||||
func (m *fakeMsg) Ack() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.acked = true
|
||||
return nil
|
||||
}
|
||||
func (m *fakeMsg) DoubleAck(context.Context) error { return nil }
|
||||
func (m *fakeMsg) Nak() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.naked = true
|
||||
return nil
|
||||
}
|
||||
func (m *fakeMsg) NakWithDelay(time.Duration) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.naked = true
|
||||
return nil
|
||||
}
|
||||
func (m *fakeMsg) InProgress() error { return nil }
|
||||
func (m *fakeMsg) Term() error { return nil }
|
||||
func (m *fakeMsg) TermWithReason(string) error { return nil }
|
||||
|
||||
func (m *fakeMsg) isAcked() bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.acked
|
||||
}
|
||||
func (m *fakeMsg) isNaked() bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.naked
|
||||
}
|
||||
|
||||
// fakePersister records calls and can be made to fail.
|
||||
type fakePersister struct {
|
||||
fail bool
|
||||
called int
|
||||
}
|
||||
|
||||
func (p *fakePersister) Store(_ context.Context, b *batcher.Batch) (StoreResult, error) {
|
||||
p.called++
|
||||
if p.fail {
|
||||
return StoreResult{}, errors.New("boom")
|
||||
}
|
||||
return StoreResult{ObjectKey: "k", Events: len(b.Items)}, nil
|
||||
}
|
||||
|
||||
func batchWith(msgs ...*fakeMsg) *batcher.Batch {
|
||||
b := &batcher.Batch{Subject: "s"}
|
||||
for _, m := range msgs {
|
||||
b.Items = append(b.Items, batcher.Item{Subject: "s", Raw: m.data, Ack: jetstream.Msg(m)})
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestFlushAcksOnlyAfterPersist is the core correctness test: messages are acked
|
||||
// exactly when Store succeeds, and Nak'd (never acked) when it fails.
|
||||
func TestFlushAcksOnSuccess(t *testing.T) {
|
||||
p := &fakePersister{}
|
||||
r := NewRunner(Options{Persister: p})
|
||||
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)}
|
||||
m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"h2"}`)}
|
||||
|
||||
r.flush(context.Background(), batchWith(m1, m2), "test")
|
||||
|
||||
if p.called != 1 {
|
||||
t.Fatalf("Store called %d times, want 1", p.called)
|
||||
}
|
||||
if !m1.isAcked() || !m2.isAcked() {
|
||||
t.Errorf("messages should be acked after successful persist")
|
||||
}
|
||||
if m1.isNaked() || m2.isNaked() {
|
||||
t.Errorf("messages must not be naked on success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushNaksOnFailure(t *testing.T) {
|
||||
p := &fakePersister{fail: true}
|
||||
r := NewRunner(Options{Persister: p})
|
||||
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)}
|
||||
|
||||
r.flush(context.Background(), batchWith(m1), "test")
|
||||
|
||||
if m1.isAcked() {
|
||||
t.Errorf("message must NOT be acked when persist fails")
|
||||
}
|
||||
if !m1.isNaked() {
|
||||
t.Errorf("message should be naked so JetStream redelivers")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushEmptyBatchNoop(t *testing.T) {
|
||||
p := &fakePersister{}
|
||||
r := NewRunner(Options{Persister: p})
|
||||
r.flush(context.Background(), &batcher.Batch{Subject: "s"}, "test")
|
||||
if p.called != 0 {
|
||||
t.Errorf("empty batch should not call Store")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRouteAndFlushIntegration wires a real batcher: adding enough messages to
|
||||
// fill the batch triggers a full flush that persists and acks exactly those.
|
||||
func TestRouteFlushViaBatcher(t *testing.T) {
|
||||
p := &fakePersister{}
|
||||
bat := batcher.New(batcher.Limits{MaxEvents: 2})
|
||||
r := NewRunner(Options{Persister: p, Batcher: bat})
|
||||
|
||||
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"a"}`)}
|
||||
m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"b"}`)}
|
||||
r.route(context.Background(), m1)
|
||||
if m1.isAcked() {
|
||||
t.Errorf("first message should not be acked before batch fills")
|
||||
}
|
||||
r.route(context.Background(), m2) // fills batch -> flush
|
||||
|
||||
if p.called != 1 {
|
||||
t.Fatalf("Store called %d times, want 1 after fill", p.called)
|
||||
}
|
||||
if !m1.isAcked() || !m2.isAcked() {
|
||||
t.Errorf("both messages should be acked after the full-batch flush")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
)
|
||||
|
||||
// genTestKey creates an OpenPGP keypair, returns the armored public key (as the
|
||||
// service would export from Vault) and an unwrap func that decrypts the wrapped
|
||||
// DEK with the private key — simulating the Vault GPG engine's decrypt endpoint
|
||||
// (which returns the plaintext of a whole OpenPGP message).
|
||||
func genTestKey(t *testing.T) (armoredPub []byte, unwrap UnwrapFunc) {
|
||||
t.Helper()
|
||||
ent, err := openpgp.NewEntity("logarchiver-test", "unit test", "test@unkin.net", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEntity: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("armor encode: %v", err)
|
||||
}
|
||||
if err := ent.Serialize(w); err != nil {
|
||||
t.Fatalf("serialize public: %v", err)
|
||||
}
|
||||
_ = w.Close()
|
||||
|
||||
unwrap = func(wrapped []byte) ([]byte, error) {
|
||||
md, err := openpgp.ReadMessage(bytes.NewReader(wrapped), openpgp.EntityList{ent}, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(md.UnverifiedBody)
|
||||
}
|
||||
return buf.Bytes(), unwrap
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, err := LoadPublicKey(armoredPub)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublicKey: %v", err)
|
||||
}
|
||||
if len(pub.Fingerprint) != 40 {
|
||||
t.Errorf("fingerprint = %q, want 40 hex chars", pub.Fingerprint)
|
||||
}
|
||||
if pub.Fingerprint != strings.ToUpper(pub.Fingerprint) {
|
||||
t.Errorf("fingerprint should be uppercase: %q", pub.Fingerprint)
|
||||
}
|
||||
|
||||
// A multi-line NDJSON payload larger than the frame size (forces >1 frame).
|
||||
var payload bytes.Buffer
|
||||
for i := 0; i < 5000; i++ {
|
||||
payload.WriteString(`{"host":"node-1","message":"line `)
|
||||
payload.WriteString(strings.Repeat("x", 50))
|
||||
payload.WriteString(`"}` + "\n")
|
||||
}
|
||||
plaintext := payload.Bytes()
|
||||
|
||||
var sealed bytes.Buffer
|
||||
res, err := Seal(&sealed, plaintext, pub, "logarchive", 4096)
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if res.RawBytes != int64(len(plaintext)) {
|
||||
t.Errorf("RawBytes = %d, want %d", res.RawBytes, len(plaintext))
|
||||
}
|
||||
if int64(sealed.Len()) != res.StoredBytes {
|
||||
t.Errorf("StoredBytes = %d, buffer = %d", res.StoredBytes, sealed.Len())
|
||||
}
|
||||
// Compression should shrink this highly repetitive payload.
|
||||
if res.StoredBytes >= res.RawBytes {
|
||||
t.Errorf("stored (%d) not smaller than raw (%d)", res.StoredBytes, res.RawBytes)
|
||||
}
|
||||
// Design property: only a tiny wrapped DEK goes to Vault, regardless of size.
|
||||
if res.Header.WrappedDEKLen > 4096 {
|
||||
t.Errorf("wrapped DEK unexpectedly large: %d bytes", res.Header.WrappedDEKLen)
|
||||
}
|
||||
if res.Header.KeyFingerprint != pub.Fingerprint {
|
||||
t.Errorf("header fingerprint mismatch")
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if !bytes.Equal(out.Bytes(), plaintext) {
|
||||
t.Fatalf("round-trip mismatch: got %d bytes, want %d", out.Len(), len(plaintext))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripEmpty(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte{}, pub, "k", 4096); err != nil {
|
||||
t.Fatalf("Seal empty: %v", err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
||||
t.Fatalf("Open empty: %v", err)
|
||||
}
|
||||
if out.Len() != 0 {
|
||||
t.Errorf("empty round-trip produced %d bytes", out.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTamperDetected(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte("hello world\n"), pub, "k", 4096); err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
data := sealed.Bytes()
|
||||
// Flip a byte near the end (inside a frame's ciphertext/tag).
|
||||
data[len(data)-3] ^= 0xff
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(data), &out, unwrap); err == nil {
|
||||
t.Fatalf("expected GCM authentication failure on tampered ciphertext")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadMagic(t *testing.T) {
|
||||
_, _, err := ReadHeader(bytes.NewReader([]byte("NOTLARC.....")))
|
||||
if err == nil {
|
||||
t.Fatalf("expected bad-magic error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHeaderFields(t *testing.T) {
|
||||
armoredPub, _ := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte("x\n"), pub, "logarchive", 4096); err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
hdr, wrapped, err := ReadHeader(bytes.NewReader(sealed.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadHeader: %v", err)
|
||||
}
|
||||
if hdr.KeyName != "logarchive" {
|
||||
t.Errorf("KeyName = %q", hdr.KeyName)
|
||||
}
|
||||
if hdr.Compression != "zstd" || hdr.Cipher != "AES-256-GCM" {
|
||||
t.Errorf("algo metadata wrong: %+v", hdr)
|
||||
}
|
||||
if len(wrapped) != hdr.WrappedDEKLen {
|
||||
t.Errorf("wrapped len %d != header %d", len(wrapped), hdr.WrappedDEKLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDigestArmoredStable(t *testing.T) {
|
||||
// Guards the test helper used elsewhere; identical input -> identical digest.
|
||||
a := digestArmored([]byte("abc"))
|
||||
b := digestArmored([]byte("abc"))
|
||||
if a != b || a == "" {
|
||||
t.Errorf("digestArmored not stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// ContainerMagic identifies a logarchiver object and its container version.
|
||||
var ContainerMagic = []byte("LARC1\n")
|
||||
|
||||
const (
|
||||
dekSize = 32 // AES-256
|
||||
noncePrefixSize = 4
|
||||
counterSize = 8
|
||||
// maxFrameCiphertext bounds a single frame read to avoid unbounded allocation
|
||||
// from a corrupt/hostile length prefix.
|
||||
maxFrameCiphertext = 128 << 20
|
||||
)
|
||||
|
||||
// Header is the LARC1 object header (JSON), written after the magic and a
|
||||
// uint32 big-endian length prefix.
|
||||
type Header struct {
|
||||
Version int `json:"v"`
|
||||
KeyName string `json:"key_name"`
|
||||
KeyFingerprint string `json:"key_fingerprint"`
|
||||
WrappedDEKLen int `json:"wrapped_dek_len"`
|
||||
NoncePrefix []byte `json:"nonce_prefix"` // base64 in JSON
|
||||
FrameSize int `json:"frame_size"` // plaintext (compressed) bytes per frame
|
||||
Compression string `json:"compression"` // "zstd"
|
||||
Cipher string `json:"cipher"` // "AES-256-GCM"
|
||||
}
|
||||
|
||||
// SealResult reports what Seal produced (for the index row).
|
||||
type SealResult struct {
|
||||
Header Header
|
||||
RawBytes int64 // input NDJSON length
|
||||
StoredBytes int64 // full container length
|
||||
}
|
||||
|
||||
// Seal compresses plaintext with zstd, encrypts it under a fresh random DEK
|
||||
// using framed AES-256-GCM, wraps the DEK to pub with OpenPGP, and writes the
|
||||
// LARC1 container to w. keyName is recorded in the header for operator context.
|
||||
func Seal(w io.Writer, plaintext []byte, pub *PublicKey, keyName string, frameSize int) (SealResult, error) {
|
||||
if pub == nil {
|
||||
return SealResult{}, fmt.Errorf("nil public key")
|
||||
}
|
||||
if frameSize <= 0 {
|
||||
frameSize = 1 << 20
|
||||
}
|
||||
|
||||
// 1. Compress.
|
||||
enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("zstd writer: %w", err)
|
||||
}
|
||||
compressed := enc.EncodeAll(plaintext, nil)
|
||||
_ = enc.Close()
|
||||
|
||||
// 2. DEK + nonce prefix.
|
||||
dek := make([]byte, dekSize)
|
||||
if _, err := rand.Read(dek); err != nil {
|
||||
return SealResult{}, fmt.Errorf("gen dek: %w", err)
|
||||
}
|
||||
noncePrefix := make([]byte, noncePrefixSize)
|
||||
if _, err := rand.Read(noncePrefix); err != nil {
|
||||
return SealResult{}, fmt.Errorf("gen nonce prefix: %w", err)
|
||||
}
|
||||
|
||||
// 3. Wrap the DEK to the public key (small standard OpenPGP message).
|
||||
wrapped, err := wrapDEK(dek, pub)
|
||||
if err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("aes cipher: %w", err)
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("gcm: %w", err)
|
||||
}
|
||||
|
||||
hdr := Header{
|
||||
Version: 1,
|
||||
KeyName: keyName,
|
||||
KeyFingerprint: pub.Fingerprint,
|
||||
WrappedDEKLen: len(wrapped),
|
||||
NoncePrefix: noncePrefix,
|
||||
FrameSize: frameSize,
|
||||
Compression: "zstd",
|
||||
Cipher: "AES-256-GCM",
|
||||
}
|
||||
hdrJSON, err := json.Marshal(hdr)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("marshal header: %w", err)
|
||||
}
|
||||
|
||||
cw := &countingWriter{w: w}
|
||||
|
||||
// magic
|
||||
if _, err := cw.Write(ContainerMagic); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
// header length + header
|
||||
if err := writeUint32(cw, uint32(len(hdrJSON))); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
if _, err := cw.Write(hdrJSON); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
// wrapped DEK
|
||||
if _, err := cw.Write(wrapped); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
// frames
|
||||
for i, off := 0, 0; off < len(compressed); i++ {
|
||||
end := off + frameSize
|
||||
if end > len(compressed) {
|
||||
end = len(compressed)
|
||||
}
|
||||
nonce := frameNonce(noncePrefix, uint64(i))
|
||||
aad := aadFor(uint64(i))
|
||||
ct := gcm.Seal(nil, nonce, compressed[off:end], aad)
|
||||
if err := writeUint32(cw, uint32(len(ct))); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
if _, err := cw.Write(ct); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
off = end
|
||||
}
|
||||
// terminating zero-length frame
|
||||
if err := writeUint32(cw, 0); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
return SealResult{Header: hdr, RawBytes: int64(len(plaintext)), StoredBytes: cw.n}, nil
|
||||
}
|
||||
|
||||
// wrapDEK OpenPGP-encrypts the DEK to pub, producing a compact binary message.
|
||||
func wrapDEK(dek []byte, pub *PublicKey) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
cfg := &packet.Config{
|
||||
DefaultCipher: packet.CipherAES256,
|
||||
}
|
||||
wc, err := openpgp.Encrypt(&buf, []*openpgp.Entity{pub.entity}, nil, nil, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("openpgp encrypt dek: %w", err)
|
||||
}
|
||||
if _, err := wc.Write(dek); err != nil {
|
||||
return nil, fmt.Errorf("write dek: %w", err)
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close openpgp: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnwrapFunc recovers the DEK from the wrapped OpenPGP blob. In production this
|
||||
// calls the Vault GPG engine decrypt endpoint; tests supply a local one.
|
||||
type UnwrapFunc func(wrappedDEK []byte) (dek []byte, err error)
|
||||
|
||||
// ReadHeader reads and validates the LARC1 magic + header and the wrapped DEK,
|
||||
// leaving r positioned at the first frame. It does not require decryption keys,
|
||||
// so it is cheap for `search`/metadata inspection.
|
||||
func ReadHeader(r io.Reader) (Header, []byte, error) {
|
||||
magic := make([]byte, len(ContainerMagic))
|
||||
if _, err := io.ReadFull(r, magic); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read magic: %w", err)
|
||||
}
|
||||
if !bytes.Equal(magic, ContainerMagic) {
|
||||
return Header{}, nil, fmt.Errorf("bad magic: not a logarchiver (LARC1) object")
|
||||
}
|
||||
hlen, err := readUint32(r)
|
||||
if err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read header len: %w", err)
|
||||
}
|
||||
if hlen == 0 || hlen > 1<<20 {
|
||||
return Header{}, nil, fmt.Errorf("implausible header length %d", hlen)
|
||||
}
|
||||
hdrJSON := make([]byte, hlen)
|
||||
if _, err := io.ReadFull(r, hdrJSON); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read header: %w", err)
|
||||
}
|
||||
var hdr Header
|
||||
if err := json.Unmarshal(hdrJSON, &hdr); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("parse header: %w", err)
|
||||
}
|
||||
if hdr.Version != 1 {
|
||||
return Header{}, nil, fmt.Errorf("unsupported container version %d", hdr.Version)
|
||||
}
|
||||
if hdr.WrappedDEKLen <= 0 || hdr.WrappedDEKLen > 1<<20 {
|
||||
return Header{}, nil, fmt.Errorf("implausible wrapped dek length %d", hdr.WrappedDEKLen)
|
||||
}
|
||||
wrapped := make([]byte, hdr.WrappedDEKLen)
|
||||
if _, err := io.ReadFull(r, wrapped); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read wrapped dek: %w", err)
|
||||
}
|
||||
return hdr, wrapped, nil
|
||||
}
|
||||
|
||||
// Open reads a LARC1 container from r, recovers the DEK via unwrap, and streams
|
||||
// the decrypted, decompressed NDJSON to w.
|
||||
func Open(r io.Reader, w io.Writer, unwrap UnwrapFunc) error {
|
||||
hdr, wrapped, err := ReadHeader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dek, err := unwrap(wrapped)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unwrap dek: %w", err)
|
||||
}
|
||||
if len(dek) != dekSize {
|
||||
return fmt.Errorf("unwrapped dek has wrong length %d", len(dek))
|
||||
}
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aes cipher: %w", err)
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gcm: %w", err)
|
||||
}
|
||||
|
||||
fr := &frameReader{r: r, gcm: gcm, noncePrefix: hdr.NoncePrefix}
|
||||
zr, err := zstd.NewReader(fr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
if _, err := io.Copy(w, zr); err != nil {
|
||||
return fmt.Errorf("decompress: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// frameReader decrypts LARC1 frames on demand, presenting the decrypted
|
||||
// (compressed) bytes as an io.Reader for the zstd decoder.
|
||||
type frameReader struct {
|
||||
r io.Reader
|
||||
gcm cipher.AEAD
|
||||
noncePrefix []byte
|
||||
counter uint64
|
||||
buf []byte // leftover decrypted plaintext not yet consumed
|
||||
done bool
|
||||
}
|
||||
|
||||
func (f *frameReader) Read(p []byte) (int, error) {
|
||||
if len(f.buf) == 0 && !f.done {
|
||||
if err := f.next(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if len(f.buf) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, f.buf)
|
||||
f.buf = f.buf[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (f *frameReader) next() error {
|
||||
ln, err := readUint32(f.r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read frame len: %w", err)
|
||||
}
|
||||
if ln == 0 { // terminator
|
||||
f.done = true
|
||||
return nil
|
||||
}
|
||||
if ln > maxFrameCiphertext {
|
||||
return fmt.Errorf("frame length %d exceeds max", ln)
|
||||
}
|
||||
ct := make([]byte, ln)
|
||||
if _, err := io.ReadFull(f.r, ct); err != nil {
|
||||
return fmt.Errorf("read frame: %w", err)
|
||||
}
|
||||
nonce := frameNonce(f.noncePrefix, f.counter)
|
||||
aad := aadFor(f.counter)
|
||||
pt, err := f.gcm.Open(nil, nonce, ct, aad)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt frame %d: %w", f.counter, err)
|
||||
}
|
||||
f.counter++
|
||||
f.buf = pt
|
||||
return nil
|
||||
}
|
||||
|
||||
func frameNonce(prefix []byte, counter uint64) []byte {
|
||||
nonce := make([]byte, noncePrefixSize+counterSize)
|
||||
copy(nonce, prefix)
|
||||
binary.BigEndian.PutUint64(nonce[noncePrefixSize:], counter)
|
||||
return nonce
|
||||
}
|
||||
|
||||
func aadFor(counter uint64) []byte {
|
||||
aad := make([]byte, counterSize)
|
||||
binary.BigEndian.PutUint64(aad, counter)
|
||||
return aad
|
||||
}
|
||||
|
||||
func writeUint32(w io.Writer, v uint32) error {
|
||||
var b [4]byte
|
||||
binary.BigEndian.PutUint32(b[:], v)
|
||||
_, err := w.Write(b[:])
|
||||
return err
|
||||
}
|
||||
|
||||
func readUint32(r io.Reader) (uint32, error) {
|
||||
var b [4]byte
|
||||
if _, err := io.ReadFull(r, b[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.BigEndian.Uint32(b[:]), nil
|
||||
}
|
||||
|
||||
type countingWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Package crypto implements logarchiver's object encryption.
|
||||
//
|
||||
// # Why not plain OpenPGP-encrypt the whole object?
|
||||
//
|
||||
// The private key lives only in Ben's Vault GPG secrets engine
|
||||
// (vault-plugin-secrets-gpg). That engine's decrypt endpoint does WHOLE-payload
|
||||
// inline decryption only: you POST the entire OpenPGP message (base64 in a JSON
|
||||
// body) and it returns the entire plaintext (base64). There is no session-key /
|
||||
// PKESK extraction and no streaming, so a multi-hundred-MiB archive could not be
|
||||
// retrieved without blowing Vault's request-size limit and buffering everything
|
||||
// twice in the server.
|
||||
//
|
||||
// # The wrapped-DEK envelope (container "LARC1")
|
||||
//
|
||||
// logarchiver therefore does hybrid encryption itself:
|
||||
//
|
||||
// - a fresh random 256-bit Data Encryption Key (DEK) per object;
|
||||
// - the bulk (zstd-compressed NDJSON) is encrypted locally with AES-256-GCM in
|
||||
// independent frames, so decryption streams frame-by-frame;
|
||||
// - only the 32-byte DEK is OpenPGP-encrypted to the engine's PUBLIC key,
|
||||
// producing a small (~hundreds of bytes) standard OpenPGP message.
|
||||
//
|
||||
// On retrieval the CLI sends ONLY that small wrapped-DEK blob to the engine's
|
||||
// decrypt endpoint, recovers the DEK, and streams the bulk locally. The Vault
|
||||
// round-trip is tiny and constant regardless of object size, and the private key
|
||||
// never leaves Vault. The trade-off vs. a single standard OpenPGP object: these
|
||||
// objects are a logarchiver-specific container, not decryptable by a bare `gpg`
|
||||
// even with the private key. The retrieval runbook documents the format.
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
)
|
||||
|
||||
// PublicKey is a parsed OpenPGP public key plus its fingerprint (uppercase hex,
|
||||
// no spaces — matching the Vault GPG engine's `%X` fingerprint format).
|
||||
type PublicKey struct {
|
||||
entity *openpgp.Entity
|
||||
Fingerprint string
|
||||
}
|
||||
|
||||
// LoadPublicKey parses an ASCII-armored (or binary) OpenPGP public key.
|
||||
func LoadPublicKey(data []byte) (*PublicKey, error) {
|
||||
var keyring openpgp.EntityList
|
||||
var err error
|
||||
if strings.Contains(string(data), "BEGIN PGP") {
|
||||
block, berr := armor.Decode(strings.NewReader(string(data)))
|
||||
if berr != nil {
|
||||
return nil, fmt.Errorf("decode armor: %w", berr)
|
||||
}
|
||||
keyring, err = openpgp.ReadKeyRing(block.Body)
|
||||
} else {
|
||||
keyring, err = openpgp.ReadKeyRing(strings.NewReader(string(data)))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read public key: %w", err)
|
||||
}
|
||||
if len(keyring) == 0 {
|
||||
return nil, fmt.Errorf("no public key found")
|
||||
}
|
||||
ent := keyring[0]
|
||||
if ent.PrimaryKey == nil {
|
||||
return nil, fmt.Errorf("key has no primary public key")
|
||||
}
|
||||
return &PublicKey{
|
||||
entity: ent,
|
||||
Fingerprint: fmt.Sprintf("%X", ent.PrimaryKey.Fingerprint),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// digestArmored is used by tests to sanity check key identity independent of
|
||||
// go-crypto internals.
|
||||
func digestArmored(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", sum[:8])
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Package event extracts the fields logarchiver needs (host, timestamp) from a
|
||||
// raw log event as it flows through the centralized logging JetStream stream
|
||||
// (argocd-apps #296). Events are one JSON object per NATS message and are NOT
|
||||
// normalized — logarchiver persists them raw, so extraction must be tolerant of
|
||||
// the two shapes that share the logs.> subject space:
|
||||
//
|
||||
// - k8s pod logs (Vector kubernetes_logs): host lives at .kubernetes.pod_node_name,
|
||||
// timestamp at .timestamp (RFC3339).
|
||||
// - VM logs (vm-ingest): host at .host (fallback .hostname), timestamp at
|
||||
// .timestamp (fallback .ts).
|
||||
//
|
||||
// The NATS subject itself is authoritative for partitioning and is supplied by
|
||||
// the consumer, not read from the payload.
|
||||
package event
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Meta is the minimal, index-relevant projection of a raw log event.
|
||||
type Meta struct {
|
||||
// Host is the best-effort source host/node for the event, or "" if none
|
||||
// could be determined.
|
||||
Host string
|
||||
// Timestamp is the event time. Ok reports whether a timestamp field was
|
||||
// found and parsed; when false callers should fall back to ingest time.
|
||||
Timestamp time.Time
|
||||
Ok bool
|
||||
}
|
||||
|
||||
// hostPaths and tsPaths are tried in order. Dotted paths descend into nested
|
||||
// objects (only .kubernetes.pod_node_name is nested today).
|
||||
var (
|
||||
hostPaths = [][]string{
|
||||
{"host"},
|
||||
{"hostname"},
|
||||
{"kubernetes", "pod_node_name"},
|
||||
}
|
||||
tsPaths = [][]string{
|
||||
{"timestamp"},
|
||||
{"ts"},
|
||||
{"@timestamp"},
|
||||
}
|
||||
)
|
||||
|
||||
// Extract parses raw (a single JSON log event) and returns its host/timestamp
|
||||
// projection. It never errors: malformed or field-less events yield a zero-value
|
||||
// Meta (Host=="", Ok==false) so the archiver still stores the raw bytes and the
|
||||
// caller can fall back to ingest time. Only the fields of interest are decoded.
|
||||
func Extract(raw []byte) Meta {
|
||||
var doc map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return Meta{}
|
||||
}
|
||||
m := Meta{}
|
||||
m.Host = firstString(doc, hostPaths)
|
||||
if ts, ok := firstTime(doc, tsPaths); ok {
|
||||
m.Timestamp = ts
|
||||
m.Ok = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// firstString walks each path and returns the first value that decodes to a
|
||||
// non-empty string.
|
||||
func firstString(doc map[string]json.RawMessage, paths [][]string) string {
|
||||
for _, p := range paths {
|
||||
if v, ok := lookup(doc, p); ok {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil && s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstTime walks each path and returns the first value that parses as a
|
||||
// timestamp (RFC3339/RFC3339Nano string, or a numeric unix seconds/millis).
|
||||
func firstTime(doc map[string]json.RawMessage, paths [][]string) (time.Time, bool) {
|
||||
for _, p := range paths {
|
||||
v, ok := lookup(doc, p)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil && s != "" {
|
||||
if t, err := parseTimeString(s); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
var n json.Number
|
||||
if json.Unmarshal(v, &n) == nil {
|
||||
if t, ok := parseNumericTime(n); ok {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// lookup descends doc following path. Intermediate elements must be JSON objects.
|
||||
func lookup(doc map[string]json.RawMessage, path []string) (json.RawMessage, bool) {
|
||||
cur := doc
|
||||
for i, key := range path {
|
||||
v, ok := cur[key]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if i == len(path)-1 {
|
||||
return v, true
|
||||
}
|
||||
var next map[string]json.RawMessage
|
||||
if json.Unmarshal(v, &next) != nil {
|
||||
return nil, false
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var timeLayouts = []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05.999999999Z0700",
|
||||
"2006-01-02 15:04:05.999999999Z07:00",
|
||||
"2006-01-02 15:04:05",
|
||||
}
|
||||
|
||||
func parseTimeString(s string) (time.Time, error) {
|
||||
var lastErr error
|
||||
for _, l := range timeLayouts {
|
||||
t, err := time.Parse(l, s)
|
||||
if err == nil {
|
||||
return t.UTC(), nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return time.Time{}, lastErr
|
||||
}
|
||||
|
||||
// parseNumericTime interprets n as unix seconds, milliseconds, microseconds, or
|
||||
// nanoseconds based on magnitude. Fractional seconds are supported.
|
||||
func parseNumericTime(n json.Number) (time.Time, bool) {
|
||||
f, err := n.Float64()
|
||||
if err != nil || f <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
switch {
|
||||
case f >= 1e18: // nanoseconds
|
||||
return time.Unix(0, int64(f)).UTC(), true
|
||||
case f >= 1e15: // microseconds
|
||||
return time.Unix(0, int64(f*1e3)).UTC(), true
|
||||
case f >= 1e12: // milliseconds
|
||||
return time.Unix(0, int64(f*1e6)).UTC(), true
|
||||
default: // seconds (possibly fractional)
|
||||
sec := int64(f)
|
||||
nsec := int64((f - float64(sec)) * 1e9)
|
||||
return time.Unix(sec, nsec).UTC(), true
|
||||
}
|
||||
}
|
||||
|
||||
// SubjectToken sanitizes a NATS subject into a filesystem/object-key-safe token,
|
||||
// matching the logging stack's convention of replacing [^a-zA-Z0-9_.-] with '_'.
|
||||
// Dots are preserved because subjects are dot-delimited.
|
||||
func SubjectToken(subject string) string {
|
||||
if subject == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range subject {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '.', r == '-':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('_')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package event
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtractK8s(t *testing.T) {
|
||||
raw := []byte(`{"message":"hello from pod","stream":"stdout","timestamp":"2026-07-27T00:00:00Z","kubernetes":{"pod_name":"web-abc","pod_namespace":"shop","container_name":"web","pod_node_name":"node-1"},"ns_token":"shop","cont_token":"web"}`)
|
||||
m := Extract(raw)
|
||||
if m.Host != "node-1" {
|
||||
t.Errorf("host = %q, want node-1 (pod_node_name)", m.Host)
|
||||
}
|
||||
if !m.Ok {
|
||||
t.Fatalf("timestamp not parsed")
|
||||
}
|
||||
if !m.Timestamp.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("timestamp = %v", m.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractVMHostAndFallbacks(t *testing.T) {
|
||||
raw := []byte(`{"message":"sshd started","host":"vm-db-1","severity":"info","role":"database","host_token":"vm-db-1"}`)
|
||||
m := Extract(raw)
|
||||
if m.Host != "vm-db-1" {
|
||||
t.Errorf("host = %q, want vm-db-1", m.Host)
|
||||
}
|
||||
if m.Ok {
|
||||
t.Errorf("no timestamp field present; Ok should be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractHostnameFallback(t *testing.T) {
|
||||
m := Extract([]byte(`{"hostname":"legacy-box","ts":"2026-01-02T03:04:05Z"}`))
|
||||
if m.Host != "legacy-box" {
|
||||
t.Errorf("host = %q, want legacy-box (hostname fallback)", m.Host)
|
||||
}
|
||||
if !m.Ok || !m.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) {
|
||||
t.Errorf("ts fallback failed: ok=%v ts=%v", m.Ok, m.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractHostPrecedence(t *testing.T) {
|
||||
// .host wins over .kubernetes.pod_node_name when both present.
|
||||
m := Extract([]byte(`{"host":"explicit","kubernetes":{"pod_node_name":"node-x"}}`))
|
||||
if m.Host != "explicit" {
|
||||
t.Errorf("host precedence wrong: %q", m.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractNumericTimestamp(t *testing.T) {
|
||||
// unix millis
|
||||
m := Extract([]byte(`{"host":"h","timestamp":1769472000000}`))
|
||||
if !m.Ok {
|
||||
t.Fatalf("numeric millis not parsed")
|
||||
}
|
||||
if !m.Timestamp.Equal(time.Date(2026, 1, 27, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("numeric ts = %v", m.Timestamp.UTC())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMalformed(t *testing.T) {
|
||||
m := Extract([]byte(`not json`))
|
||||
if m.Host != "" || m.Ok {
|
||||
t.Errorf("malformed event should yield zero Meta, got %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEmptyHost(t *testing.T) {
|
||||
m := Extract([]byte(`{"host":"","hostname":"backup"}`))
|
||||
if m.Host != "backup" {
|
||||
t.Errorf("empty host should fall through to hostname, got %q", m.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubjectToken(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"logs.k8s.vault.audit": "logs.k8s.vault.audit",
|
||||
"logs.vm.vm-db-1": "logs.vm.vm-db-1",
|
||||
"logs.k8s.a/b.c": "logs.k8s.a_b.c",
|
||||
"": "_",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := SubjectToken(in); got != want {
|
||||
t.Errorf("SubjectToken(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package index
|
||||
|
||||
import "fmt"
|
||||
|
||||
// CreateDatabaseSQL creates the index database if absent.
|
||||
func CreateDatabaseSQL(database string) string {
|
||||
return fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database)
|
||||
}
|
||||
|
||||
// CreateTableSQL returns the DDL for the archive index table. One row is written
|
||||
// per archived S3 object. In-cluster the argocd bootstrap Job owns table
|
||||
// creation (like the logging stack's clickhouse-schema PostSync hook); this DDL
|
||||
// is also shipped as schema/archive_index.sql and applied by `logarchiver
|
||||
// init-schema`.
|
||||
//
|
||||
// PARTITION BY month of min_ts keeps partitions coarse (few objects/day).
|
||||
// ORDER BY (subject, min_ts) matches the primary search axes. A bloom_filter
|
||||
// skip index on hosts accelerates host lookups without a per-host column.
|
||||
func CreateTableSQL(database, table string) string {
|
||||
return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s
|
||||
(
|
||||
object_key String,
|
||||
bucket LowCardinality(String),
|
||||
subject LowCardinality(String),
|
||||
hosts Array(LowCardinality(String)),
|
||||
min_ts DateTime64(3),
|
||||
max_ts DateTime64(3),
|
||||
event_count UInt64,
|
||||
raw_bytes UInt64,
|
||||
stored_bytes UInt64,
|
||||
compression LowCardinality(String),
|
||||
cipher LowCardinality(String),
|
||||
container_format LowCardinality(String),
|
||||
key_name LowCardinality(String),
|
||||
key_fingerprint String,
|
||||
created_at DateTime64(3) DEFAULT now64(3),
|
||||
INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(min_ts)
|
||||
ORDER BY (subject, min_ts, object_key)`, database, table)
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
// Package index writes and queries the ClickHouse archive index — one row per
|
||||
// stored S3 object — so operators can answer "which objects hold vault logs
|
||||
// from host X between Y and Z" without scanning S3. The concrete store is
|
||||
// behind the Index interface so the archiver and CLI test against a fake.
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// Row is one archive-index record.
|
||||
type Row struct {
|
||||
ObjectKey string
|
||||
Bucket string
|
||||
Subject string
|
||||
Hosts []string
|
||||
MinTS time.Time
|
||||
MaxTS time.Time
|
||||
EventCount uint64
|
||||
RawBytes uint64
|
||||
StoredBytes uint64
|
||||
Compression string
|
||||
Cipher string
|
||||
ContainerFormat string
|
||||
KeyName string
|
||||
KeyFingerprint string
|
||||
}
|
||||
|
||||
// Result is one row returned by Search (a subset relevant to retrieval).
|
||||
type Result struct {
|
||||
ObjectKey string
|
||||
Bucket string
|
||||
Subject string
|
||||
Hosts []string
|
||||
MinTS time.Time
|
||||
MaxTS time.Time
|
||||
EventCount uint64
|
||||
RawBytes uint64
|
||||
StoredBytes uint64
|
||||
KeyName string
|
||||
KeyFingerprint string
|
||||
}
|
||||
|
||||
// Index is the archive-index surface.
|
||||
type Index interface {
|
||||
Insert(ctx context.Context, row Row) error
|
||||
Search(ctx context.Context, q SearchQuery) ([]Result, error)
|
||||
InitSchema(ctx context.Context) error
|
||||
Ping(ctx context.Context) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Config configures the ClickHouse client.
|
||||
type Config struct {
|
||||
Address string // host:port (native protocol, 9000)
|
||||
Database string
|
||||
Table string
|
||||
Username string
|
||||
Password string
|
||||
TLS bool
|
||||
}
|
||||
|
||||
// ClickHouse is the ClickHouse-backed Index.
|
||||
type ClickHouse struct {
|
||||
conn driver.Conn
|
||||
database string
|
||||
table string
|
||||
}
|
||||
|
||||
// NewClickHouse connects to ClickHouse.
|
||||
func NewClickHouse(ctx context.Context, cfg Config) (*ClickHouse, error) {
|
||||
opts := &clickhouse.Options{
|
||||
Addr: []string{cfg.Address},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
}
|
||||
if cfg.TLS {
|
||||
opts.TLS = &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
}
|
||||
conn, err := clickhouse.Open(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open clickhouse: %w", err)
|
||||
}
|
||||
ch := &ClickHouse{conn: conn, database: cfg.Database, table: cfg.Table}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// Ping verifies connectivity.
|
||||
func (c *ClickHouse) Ping(ctx context.Context) error {
|
||||
return c.conn.Ping(ctx)
|
||||
}
|
||||
|
||||
// Close closes the connection.
|
||||
func (c *ClickHouse) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// InitSchema creates the database and table if absent (idempotent).
|
||||
func (c *ClickHouse) InitSchema(ctx context.Context) error {
|
||||
if err := c.conn.Exec(ctx, CreateDatabaseSQL(c.database)); err != nil {
|
||||
return fmt.Errorf("create database: %w", err)
|
||||
}
|
||||
if err := c.conn.Exec(ctx, CreateTableSQL(c.database, c.table)); err != nil {
|
||||
return fmt.Errorf("create table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Insert writes one row.
|
||||
func (c *ClickHouse) Insert(ctx context.Context, row Row) error {
|
||||
sql := fmt.Sprintf(
|
||||
"INSERT INTO %s.%s (object_key, bucket, subject, hosts, min_ts, max_ts, event_count, raw_bytes, stored_bytes, compression, cipher, container_format, key_name, key_fingerprint) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
c.database, c.table)
|
||||
err := c.conn.Exec(ctx, sql,
|
||||
row.ObjectKey, row.Bucket, row.Subject, row.Hosts,
|
||||
row.MinTS, row.MaxTS, row.EventCount, row.RawBytes, row.StoredBytes,
|
||||
row.Compression, row.Cipher, row.ContainerFormat, row.KeyName, row.KeyFingerprint,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert index row: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search runs the parameterized query built from q.
|
||||
func (c *ClickHouse) Search(ctx context.Context, q SearchQuery) ([]Result, error) {
|
||||
sql, args := buildSearchSQL(c.database, c.table, q)
|
||||
rows, err := c.conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search index: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var out []Result
|
||||
for rows.Next() {
|
||||
var r Result
|
||||
if err := rows.Scan(
|
||||
&r.ObjectKey, &r.Bucket, &r.Subject, &r.Hosts,
|
||||
&r.MinTS, &r.MaxTS, &r.EventCount, &r.RawBytes, &r.StoredBytes,
|
||||
&r.KeyName, &r.KeyFingerprint,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan result: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate results: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SearchQuery describes an index search. Zero-valued fields are omitted.
|
||||
type SearchQuery struct {
|
||||
Subject string // NATS-style glob: '*' = one token, '>' = rest. Empty = any.
|
||||
Host string // exact, or a glob containing '*'. Empty = any.
|
||||
From time.Time // objects whose range overlaps [From,To]
|
||||
To time.Time
|
||||
Limit int
|
||||
}
|
||||
|
||||
// buildSearchSQL renders q into a parameterized ClickHouse SELECT and its args.
|
||||
// It is pure so it can be unit-tested without a database. Placeholders use the
|
||||
// clickhouse-go positional style (?), matching Query(ctx, sql, args...).
|
||||
func buildSearchSQL(database, table string, q SearchQuery) (string, []any) {
|
||||
var (
|
||||
where []string
|
||||
args []any
|
||||
)
|
||||
if q.Subject != "" {
|
||||
where = append(where, "match(subject, ?)")
|
||||
args = append(args, subjectToRegex(q.Subject))
|
||||
}
|
||||
if q.Host != "" {
|
||||
if strings.Contains(q.Host, "*") {
|
||||
where = append(where, "arrayExists(h -> match(h, ?), hosts)")
|
||||
args = append(args, hostGlobToRegex(q.Host))
|
||||
} else {
|
||||
where = append(where, "has(hosts, ?)")
|
||||
args = append(args, q.Host)
|
||||
}
|
||||
}
|
||||
if !q.From.IsZero() {
|
||||
// object overlaps the window if its max_ts is at/after From.
|
||||
where = append(where, "max_ts >= ?")
|
||||
args = append(args, q.From.UTC())
|
||||
}
|
||||
if !q.To.IsZero() {
|
||||
where = append(where, "min_ts <= ?")
|
||||
args = append(args, q.To.UTC())
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf(
|
||||
"SELECT object_key, bucket, subject, hosts, min_ts, max_ts, event_count, raw_bytes, stored_bytes, key_name, key_fingerprint FROM %s.%s",
|
||||
database, table)
|
||||
if len(where) > 0 {
|
||||
sql += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sql += " ORDER BY min_ts, object_key"
|
||||
if q.Limit > 0 {
|
||||
sql += " LIMIT ?"
|
||||
args = append(args, q.Limit)
|
||||
}
|
||||
return sql, args
|
||||
}
|
||||
|
||||
// subjectToRegex converts a NATS-style subject glob into an anchored regex for
|
||||
// ClickHouse match(). '*' matches exactly one dot-delimited token; '>' (only
|
||||
// meaningful as the final token) matches one or more trailing tokens. Literal
|
||||
// dots and regex metacharacters are escaped.
|
||||
func subjectToRegex(glob string) string {
|
||||
tokens := strings.Split(glob, ".")
|
||||
var parts []string
|
||||
for i, tok := range tokens {
|
||||
switch tok {
|
||||
case "*":
|
||||
parts = append(parts, `[^.]+`)
|
||||
case ">":
|
||||
// '>' consumes the rest; emit and stop.
|
||||
if i == 0 {
|
||||
return "^.+$"
|
||||
}
|
||||
return "^" + strings.Join(parts[:i], `\.`) + `(\..+)?$`
|
||||
default:
|
||||
parts = append(parts, regexEscape(tok))
|
||||
}
|
||||
}
|
||||
return "^" + strings.Join(parts, `\.`) + "$"
|
||||
}
|
||||
|
||||
// hostGlobToRegex converts a host glob (where '*' matches any run of
|
||||
// characters, including dots in an FQDN) into an anchored regex for match().
|
||||
func hostGlobToRegex(glob string) string {
|
||||
var b strings.Builder
|
||||
b.WriteByte('^')
|
||||
for _, seg := range strings.Split(glob, "*") {
|
||||
b.WriteString(regexEscape(seg))
|
||||
b.WriteString(".*")
|
||||
}
|
||||
// Trim the trailing ".*" added after the last segment, then anchor.
|
||||
out := strings.TrimSuffix(b.String(), ".*")
|
||||
return out + "$"
|
||||
}
|
||||
|
||||
func regexEscape(s string) string {
|
||||
const meta = `\.+*?()|[]{}^$`
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if strings.ContainsRune(meta, r) {
|
||||
b.WriteByte('\\')
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubjectToRegex(t *testing.T) {
|
||||
cases := []struct {
|
||||
glob string
|
||||
match []string
|
||||
nomatch []string
|
||||
}{
|
||||
{
|
||||
glob: "logs.vm.*",
|
||||
match: []string{"logs.vm.db-1", "logs.vm.web"},
|
||||
nomatch: []string{"logs.vm", "logs.vm.db.1", "logs.k8s.x"},
|
||||
},
|
||||
{
|
||||
glob: "logs.k8s.vault.>",
|
||||
match: []string{"logs.k8s.vault.audit", "logs.k8s.vault.a.b", "logs.k8s.vault"},
|
||||
nomatch: []string{"logs.k8s.shop.web", "logs.vm.x"},
|
||||
},
|
||||
{
|
||||
glob: "logs.vm.db-1",
|
||||
match: []string{"logs.vm.db-1"},
|
||||
nomatch: []string{"logs.vm.db-2", "logs.vm.db-1.x"},
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
re := regexp.MustCompile(subjectToRegex(c.glob))
|
||||
for _, s := range c.match {
|
||||
if !re.MatchString(s) {
|
||||
t.Errorf("%q -> %q should match %q", c.glob, re.String(), s)
|
||||
}
|
||||
}
|
||||
for _, s := range c.nomatch {
|
||||
if re.MatchString(s) {
|
||||
t.Errorf("%q -> %q should NOT match %q", c.glob, re.String(), s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchSQLFull(t *testing.T) {
|
||||
from := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
|
||||
to := time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)
|
||||
sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{
|
||||
Subject: "logs.k8s.vault.>",
|
||||
Host: "node-1",
|
||||
From: from,
|
||||
To: to,
|
||||
Limit: 50,
|
||||
})
|
||||
if !strings.Contains(sql, "FROM logs.archive_index") {
|
||||
t.Errorf("missing table: %s", sql)
|
||||
}
|
||||
for _, want := range []string{"match(subject, ?)", "has(hosts, ?)", "max_ts >= ?", "min_ts <= ?", "ORDER BY min_ts", "LIMIT ?"} {
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Errorf("sql missing %q: %s", want, sql)
|
||||
}
|
||||
}
|
||||
if len(args) != 5 {
|
||||
t.Fatalf("args = %d, want 5: %v", len(args), args)
|
||||
}
|
||||
if args[1] != "node-1" {
|
||||
t.Errorf("host arg = %v", args[1])
|
||||
}
|
||||
if args[4] != 50 {
|
||||
t.Errorf("limit arg = %v", args[4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchSQLEmpty(t *testing.T) {
|
||||
sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{})
|
||||
if strings.Contains(sql, "WHERE") {
|
||||
t.Errorf("empty query should have no WHERE: %s", sql)
|
||||
}
|
||||
if len(args) != 0 {
|
||||
t.Errorf("args = %v, want none", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchSQLHostGlob(t *testing.T) {
|
||||
sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{Host: "db-*"})
|
||||
if !strings.Contains(sql, "arrayExists(h -> match(h, ?), hosts)") {
|
||||
t.Errorf("host glob should use arrayExists/match: %s", sql)
|
||||
}
|
||||
if len(args) != 1 {
|
||||
t.Fatalf("args = %v", args)
|
||||
}
|
||||
re := regexp.MustCompile(args[0].(string))
|
||||
if !re.MatchString("db-1") || re.MatchString("web-1") {
|
||||
t.Errorf("host glob regex wrong: %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDDLContainsKeyColumns(t *testing.T) {
|
||||
ddl := CreateTableSQL("logs", "archive_index")
|
||||
for _, col := range []string{"object_key", "subject", "hosts", "min_ts", "max_ts", "event_count", "key_fingerprint", "bloom_filter"} {
|
||||
if !strings.Contains(ddl, col) {
|
||||
t.Errorf("DDL missing %q", col)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(ddl, "IF NOT EXISTS") {
|
||||
t.Errorf("DDL should be idempotent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Package metrics exposes Prometheus metrics for the logarchiver service. The
|
||||
// logging stack ships no metrics convention today, so this is the first
|
||||
// /metrics endpoint there; it is opt-in via config.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
// Metrics holds the collectors.
|
||||
type Metrics struct {
|
||||
objectsStored *prometheus.CounterVec
|
||||
eventsArchived *prometheus.CounterVec
|
||||
rawBytes *prometheus.CounterVec
|
||||
storedBytes *prometheus.CounterVec
|
||||
storeFailures *prometheus.CounterVec
|
||||
indexFailures *prometheus.CounterVec
|
||||
messagesFetched prometheus.Counter
|
||||
acksSent prometheus.Counter
|
||||
batchesFlushed *prometheus.CounterVec
|
||||
pendingEvents prometheus.Gauge
|
||||
}
|
||||
|
||||
// New registers the collectors on reg (use prometheus.DefaultRegisterer for the
|
||||
// default /metrics handler).
|
||||
func New(reg prometheus.Registerer) *Metrics {
|
||||
f := promauto.With(reg)
|
||||
return &Metrics{
|
||||
objectsStored: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_objects_stored_total",
|
||||
Help: "Objects successfully sealed, uploaded and indexed.",
|
||||
}, []string{"subject"}),
|
||||
eventsArchived: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_events_archived_total",
|
||||
Help: "Log events archived.",
|
||||
}, []string{"subject"}),
|
||||
rawBytes: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_raw_bytes_total",
|
||||
Help: "Raw NDJSON bytes archived (pre-compression).",
|
||||
}, []string{"subject"}),
|
||||
storedBytes: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_stored_bytes_total",
|
||||
Help: "Stored object bytes written to S3 (post-compression/encryption).",
|
||||
}, []string{"subject"}),
|
||||
storeFailures: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_store_failures_total",
|
||||
Help: "Failed store attempts (seal/upload); batch not acked.",
|
||||
}, []string{"subject"}),
|
||||
indexFailures: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_index_failures_total",
|
||||
Help: "Failed index writes after a successful S3 upload; batch not acked.",
|
||||
}, []string{"subject"}),
|
||||
messagesFetched: f.NewCounter(prometheus.CounterOpts{
|
||||
Name: "logarchiver_messages_fetched_total",
|
||||
Help: "JetStream messages fetched.",
|
||||
}),
|
||||
acksSent: f.NewCounter(prometheus.CounterOpts{
|
||||
Name: "logarchiver_acks_total",
|
||||
Help: "JetStream acknowledgements sent (after successful persist).",
|
||||
}),
|
||||
batchesFlushed: f.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "logarchiver_batches_flushed_total",
|
||||
Help: "Batches flushed, labelled by trigger (full|age|shutdown).",
|
||||
}, []string{"trigger"}),
|
||||
pendingEvents: f.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "logarchiver_pending_events",
|
||||
Help: "Events currently buffered in open batches (unacked).",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// ObjectStored records a successful persist.
|
||||
func (m *Metrics) ObjectStored(subject string, events int, rawBytes, storedBytes int64) {
|
||||
m.objectsStored.WithLabelValues(subject).Inc()
|
||||
m.eventsArchived.WithLabelValues(subject).Add(float64(events))
|
||||
m.rawBytes.WithLabelValues(subject).Add(float64(rawBytes))
|
||||
m.storedBytes.WithLabelValues(subject).Add(float64(storedBytes))
|
||||
}
|
||||
|
||||
// StoreFailed records a seal/upload failure.
|
||||
func (m *Metrics) StoreFailed(subject string) { m.storeFailures.WithLabelValues(subject).Inc() }
|
||||
|
||||
// IndexFailed records an index-write failure.
|
||||
func (m *Metrics) IndexFailed(subject string) { m.indexFailures.WithLabelValues(subject).Inc() }
|
||||
|
||||
// MessagesFetched records fetched messages.
|
||||
func (m *Metrics) MessagesFetched(n int) { m.messagesFetched.Add(float64(n)) }
|
||||
|
||||
// Acked records sent acknowledgements.
|
||||
func (m *Metrics) Acked(n int) { m.acksSent.Add(float64(n)) }
|
||||
|
||||
// BatchFlushed records a flush by trigger.
|
||||
func (m *Metrics) BatchFlushed(trigger string) { m.batchesFlushed.WithLabelValues(trigger).Inc() }
|
||||
|
||||
// SetPending sets the pending-events gauge.
|
||||
func (m *Metrics) SetPending(n int) { m.pendingEvents.Set(float64(n)) }
|
||||
@@ -0,0 +1,140 @@
|
||||
// Package s3store wraps object storage (Ceph RGW via the S3 API) behind an
|
||||
// interface so the archiver and CLI can be tested with a fake. Credentials come
|
||||
// from the standard AWS_* environment (the cephrgw BucketAccess secret
|
||||
// logs-archive-s3); this package only wires the custom endpoint, path-style
|
||||
// addressing, and the internal Vault-PKI CA needed for s3.ceph.unkin.net.
|
||||
package s3store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// ObjectStore is the minimal object-storage surface logarchiver needs.
|
||||
type ObjectStore interface {
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64) error
|
||||
Get(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
List(ctx context.Context, prefix string) ([]string, error)
|
||||
Bucket() string
|
||||
}
|
||||
|
||||
// Config configures the S3 client.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
Region string
|
||||
PathStyle bool
|
||||
CAFile string
|
||||
}
|
||||
|
||||
// Store is the S3-backed ObjectStore.
|
||||
type Store struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// New builds a Store, trusting CAFile (in addition to the system roots) when set.
|
||||
func New(ctx context.Context, cfg Config) (*Store, error) {
|
||||
httpClient, err := httpClientWithCA(cfg.CAFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
region := cfg.Region
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(region),
|
||||
awsconfig.WithHTTPClient(httpClient),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load aws config: %w", err)
|
||||
}
|
||||
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if cfg.Endpoint != "" {
|
||||
o.BaseEndpoint = &cfg.Endpoint
|
||||
}
|
||||
o.UsePathStyle = cfg.PathStyle
|
||||
})
|
||||
return &Store{client: client, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
|
||||
// Bucket returns the configured bucket name.
|
||||
func (s *Store) Bucket() string { return s.bucket }
|
||||
|
||||
// Put uploads body of the given size under key.
|
||||
func (s *Store) Put(ctx context.Context, key string, body io.Reader, size int64) error {
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
Body: body,
|
||||
ContentLength: &size,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("put s3://%s/%s: %w", s.bucket, key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get streams the object at key.
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
out, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get s3://%s/%s: %w", s.bucket, key, err)
|
||||
}
|
||||
return out.Body, nil
|
||||
}
|
||||
|
||||
// List returns object keys under prefix (paginated).
|
||||
func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
var keys []string
|
||||
p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||
Bucket: &s.bucket,
|
||||
Prefix: &prefix,
|
||||
})
|
||||
for p.HasMorePages() {
|
||||
page, err := p.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list s3://%s/%s: %w", s.bucket, prefix, err)
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
if obj.Key != nil {
|
||||
keys = append(keys, *obj.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func httpClientWithCA(caFile string) (*http.Client, error) {
|
||||
if caFile == "" {
|
||||
return http.DefaultClient, nil
|
||||
}
|
||||
pem, err := os.ReadFile(caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read s3 ca file %s: %w", caFile, err)
|
||||
}
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil || pool == nil {
|
||||
pool = x509.NewCertPool()
|
||||
}
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, fmt.Errorf("no certificates parsed from %s", caFile)
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
// Package vaultgpg is a thin client for Ben's Vault GPG secrets engine
|
||||
// (vault-plugin-secrets-gpg), used two ways:
|
||||
//
|
||||
// - the service reads the ARMORED PUBLIC key (GET <mount>/keys/<name>) to
|
||||
// encrypt objects locally; and
|
||||
// - the CLI decrypts a wrapped DEK (POST <mount>/decrypt/<name>) to retrieve
|
||||
// objects. The engine does whole-payload decrypt only, but logarchiver only
|
||||
// ever sends it the tiny wrapped-DEK blob, so that is a non-issue.
|
||||
//
|
||||
// Auth mirrors passv: ambient VAULT_* env (token or ~/.vault-token) for humans,
|
||||
// or the kubernetes auth method for the in-cluster service.
|
||||
package vaultgpg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
// Config configures the Vault client.
|
||||
type Config struct {
|
||||
Address string
|
||||
Mount string // e.g. "gpg"
|
||||
AuthMethod string // "token" | "kubernetes"
|
||||
K8sRole string
|
||||
K8sMount string // e.g. "k8s/au/syd1"
|
||||
K8sJWTPath string
|
||||
CAFile string
|
||||
}
|
||||
|
||||
// Client wraps the Vault API client for GPG-engine operations.
|
||||
type Client struct {
|
||||
api *vault.Client
|
||||
mount string
|
||||
}
|
||||
|
||||
// New builds a Client and authenticates per cfg.AuthMethod.
|
||||
func New(ctx context.Context, cfg Config) (*Client, error) {
|
||||
vc := vault.DefaultConfig()
|
||||
if err := vc.ReadEnvironment(); err != nil {
|
||||
return nil, fmt.Errorf("read vault env: %w", err)
|
||||
}
|
||||
if cfg.Address != "" {
|
||||
vc.Address = cfg.Address
|
||||
}
|
||||
if cfg.CAFile != "" {
|
||||
if err := vc.ConfigureTLS(&vault.TLSConfig{CACert: cfg.CAFile}); err != nil {
|
||||
return nil, fmt.Errorf("configure vault tls: %w", err)
|
||||
}
|
||||
}
|
||||
api, err := vault.NewClient(vc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new vault client: %w", err)
|
||||
}
|
||||
mount := cfg.Mount
|
||||
if mount == "" {
|
||||
mount = "gpg"
|
||||
}
|
||||
c := &Client{api: api, mount: strings.Trim(mount, "/")}
|
||||
|
||||
switch cfg.AuthMethod {
|
||||
case "", "token":
|
||||
if api.Token() == "" {
|
||||
tok, err := resolveToken()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
api.SetToken(tok)
|
||||
}
|
||||
case "kubernetes":
|
||||
if err := c.k8sLogin(ctx, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown vault auth_method %q", cfg.AuthMethod)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Client) k8sLogin(ctx context.Context, cfg Config) error {
|
||||
jwtPath := cfg.K8sJWTPath
|
||||
if jwtPath == "" {
|
||||
jwtPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
}
|
||||
jwt, err := os.ReadFile(jwtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read service account token: %w", err)
|
||||
}
|
||||
mount := cfg.K8sMount
|
||||
if mount == "" {
|
||||
mount = "kubernetes"
|
||||
}
|
||||
path := fmt.Sprintf("auth/%s/login", strings.Trim(mount, "/"))
|
||||
secret, err := c.api.Logical().WriteWithContext(ctx, path, map[string]any{
|
||||
"role": cfg.K8sRole,
|
||||
"jwt": strings.TrimSpace(string(jwt)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("kubernetes login: %w", err)
|
||||
}
|
||||
if secret == nil || secret.Auth == nil || secret.Auth.ClientToken == "" {
|
||||
return fmt.Errorf("kubernetes login returned no token")
|
||||
}
|
||||
c.api.SetToken(secret.Auth.ClientToken)
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveToken mirrors passv: VAULT_TOKEN, else ~/.vault-token.
|
||||
func resolveToken() (string, error) {
|
||||
if t := os.Getenv("VAULT_TOKEN"); t != "" {
|
||||
return t, nil
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no VAULT_TOKEN and cannot find home dir: %w", err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(home, ".vault-token"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no VAULT_TOKEN and no ~/.vault-token: %w", err)
|
||||
}
|
||||
tok := strings.TrimSpace(string(data))
|
||||
if tok == "" {
|
||||
return "", fmt.Errorf("~/.vault-token is empty")
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
// PublicKey is the result of reading a GPG engine key.
|
||||
type PublicKey struct {
|
||||
Armored string
|
||||
Fingerprint string
|
||||
}
|
||||
|
||||
// FetchPublicKey reads <mount>/keys/<name> and returns the latest armored public
|
||||
// key and its fingerprint.
|
||||
func (c *Client) FetchPublicKey(ctx context.Context, name string) (PublicKey, error) {
|
||||
path := fmt.Sprintf("%s/keys/%s", c.mount, name)
|
||||
secret, err := c.api.Logical().ReadWithContext(ctx, path)
|
||||
if err != nil {
|
||||
return PublicKey{}, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
if secret == nil || secret.Data == nil {
|
||||
return PublicKey{}, fmt.Errorf("key %q not found at %s", name, path)
|
||||
}
|
||||
pub, _ := secret.Data["public_key"].(string)
|
||||
if pub == "" {
|
||||
return PublicKey{}, fmt.Errorf("key %q has no public_key field", name)
|
||||
}
|
||||
fpr, _ := secret.Data["fingerprint"].(string)
|
||||
return PublicKey{Armored: pub, Fingerprint: fpr}, nil
|
||||
}
|
||||
|
||||
// Decrypt sends ciphertext (raw binary OpenPGP) to <mount>/decrypt/<name> and
|
||||
// returns the plaintext. The engine auto-detects binary vs armored; we send
|
||||
// base64 of the raw bytes, as passv does.
|
||||
func (c *Client) Decrypt(ctx context.Context, name string, ciphertext []byte) ([]byte, error) {
|
||||
path := fmt.Sprintf("%s/decrypt/%s", c.mount, name)
|
||||
secret, err := c.api.Logical().WriteWithContext(ctx, path, map[string]any{
|
||||
"ciphertext": base64.StdEncoding.EncodeToString(ciphertext),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt via %s: %w", path, err)
|
||||
}
|
||||
if secret == nil || secret.Data == nil {
|
||||
return nil, fmt.Errorf("decrypt returned no data")
|
||||
}
|
||||
b64, _ := secret.Data["plaintext"].(string)
|
||||
if b64 == "" {
|
||||
return nil, fmt.Errorf("decrypt returned no plaintext")
|
||||
}
|
||||
plain, err := base64.StdEncoding.DecodeString(b64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode plaintext: %w", err)
|
||||
}
|
||||
return plain, nil
|
||||
}
|
||||
|
||||
// TokenTTL returns the remaining lease TTL of the current token, for diagnostics.
|
||||
func (c *Client) TokenTTL(ctx context.Context) (time.Duration, error) {
|
||||
secret, err := c.api.Auth().Token().LookupSelfWithContext(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ttl, err := secret.TokenTTL()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ttl, nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
---
|
||||
# nfpm config for building the logarchiver RPM (CLI use-case).
|
||||
# Rendered through envsubst (see scripts/build-rpm.sh) then fed to `nfpm pkg`.
|
||||
|
||||
name: ${PACKAGE_NAME}
|
||||
version: ${PACKAGE_VERSION}
|
||||
release: ${PACKAGE_RELEASE}
|
||||
arch: ${PACKAGE_ARCH}
|
||||
platform: ${PACKAGE_PLATFORM}
|
||||
section: default
|
||||
priority: extra
|
||||
description: "${PACKAGE_DESCRIPTION}"
|
||||
|
||||
maintainer: ${PACKAGE_MAINTAINER}
|
||||
homepage: ${PACKAGE_HOMEPAGE}
|
||||
license: ${PACKAGE_LICENSE}
|
||||
|
||||
disable_globbing: false
|
||||
|
||||
replaces:
|
||||
- logarchiver
|
||||
provides:
|
||||
- logarchiver
|
||||
|
||||
contents:
|
||||
- src: dist/logarchiver
|
||||
dst: /usr/bin/logarchiver
|
||||
file_info:
|
||||
mode: 0755
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
# Shell completions (generated by scripts/build-rpm.sh before packaging).
|
||||
- src: dist/completions/logarchiver.bash
|
||||
dst: /usr/share/bash-completion/completions/logarchiver
|
||||
file_info:
|
||||
mode: 0644
|
||||
- src: dist/completions/_logarchiver
|
||||
dst: /usr/share/zsh/site-functions/_logarchiver
|
||||
file_info:
|
||||
mode: 0644
|
||||
- src: dist/completions/logarchiver.fish
|
||||
dst: /usr/share/fish/vendor_completions.d/logarchiver.fish
|
||||
file_info:
|
||||
mode: 0644
|
||||
|
||||
# Example config (documentation; not wired into any service by default).
|
||||
- src: config.example.yaml
|
||||
dst: /usr/share/doc/logarchiver/config.example.yaml
|
||||
type: config
|
||||
file_info:
|
||||
mode: 0644
|
||||
@@ -0,0 +1,35 @@
|
||||
-- logarchiver archive index schema.
|
||||
--
|
||||
-- One row is written per stored S3 object. In-cluster this DDL is owned by the
|
||||
-- argocd bootstrap Job (a ClickHouse PostSync hook, like the logging stack's
|
||||
-- clickhouse-schema job); `logarchiver init-schema` applies the same statements
|
||||
-- for local/dev use, and `logarchiver init-schema --print` emits them.
|
||||
--
|
||||
-- Keep this file in sync with internal/index/ddl.go (the source of truth used by
|
||||
-- init-schema). The database/table names below match the config defaults
|
||||
-- (database `logs`, table `archive_index`).
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS logs;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs.archive_index
|
||||
(
|
||||
object_key String, -- S3 key of the stored object
|
||||
bucket LowCardinality(String), -- S3 bucket (e.g. logs-archive)
|
||||
subject LowCardinality(String), -- NATS subject the batch came from
|
||||
hosts Array(LowCardinality(String)),-- distinct source hosts in the object
|
||||
min_ts DateTime64(3), -- earliest event time in the object
|
||||
max_ts DateTime64(3), -- latest event time in the object
|
||||
event_count UInt64, -- number of events
|
||||
raw_bytes UInt64, -- pre-compression NDJSON bytes
|
||||
stored_bytes UInt64, -- stored object bytes (post zstd+encrypt)
|
||||
compression LowCardinality(String), -- 'zstd'
|
||||
cipher LowCardinality(String), -- 'AES-256-GCM'
|
||||
container_format LowCardinality(String), -- 'LARC1'
|
||||
key_name LowCardinality(String), -- Vault GPG engine key name
|
||||
key_fingerprint String, -- 40-hex OpenPGP fingerprint (engine %X)
|
||||
created_at DateTime64(3) DEFAULT now64(3),
|
||||
INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(min_ts)
|
||||
ORDER BY (subject, min_ts, object_key);
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Package the (already built) logarchiver binary into an RPM with nfpm,
|
||||
# bundling generated bash/zsh/fish shell completions.
|
||||
# Usage: scripts/build-rpm.sh [version] (version defaults to $CI_COMMIT_TAG)
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${ROOT_DIR}"
|
||||
|
||||
VERSION="${1:-${CI_COMMIT_TAG:-0.0.0-dev}}"
|
||||
VERSION="${VERSION#v}" # strip a leading v
|
||||
BINARY="logarchiver"
|
||||
DIST="dist"
|
||||
|
||||
if [ ! -f "${DIST}/${BINARY}" ]; then
|
||||
echo "ERROR: ${DIST}/${BINARY} not found; run 'make build' first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate shell completions from the freshly built binary so they always match
|
||||
# the shipped flags/subcommands.
|
||||
COMP_DIR="${DIST}/completions"
|
||||
mkdir -p "${COMP_DIR}"
|
||||
"./${DIST}/${BINARY}" completion bash >"${COMP_DIR}/${BINARY}.bash"
|
||||
"./${DIST}/${BINARY}" completion zsh >"${COMP_DIR}/_${BINARY}"
|
||||
"./${DIST}/${BINARY}" completion fish >"${COMP_DIR}/${BINARY}.fish"
|
||||
|
||||
export PACKAGE_NAME="${BINARY}"
|
||||
export PACKAGE_VERSION="${VERSION}"
|
||||
export PACKAGE_RELEASE="1"
|
||||
export PACKAGE_ARCH="amd64"
|
||||
export PACKAGE_PLATFORM="linux"
|
||||
export PACKAGE_DESCRIPTION="Archive NATS JetStream logs to S3 (zstd + OpenPGP, indexed) and search/retrieve them. Service + operator CLI."
|
||||
export PACKAGE_MAINTAINER="Ben Vincent <ben@unkin.net>"
|
||||
export PACKAGE_HOMEPAGE="https://git.unkin.net/unkin/logarchiver"
|
||||
export PACKAGE_LICENSE="MIT"
|
||||
|
||||
envsubst <packaging/nfpm.yaml >"${DIST}/nfpm.yaml"
|
||||
nfpm pkg --config "${DIST}/nfpm.yaml" --target "${DIST}" --packager rpm
|
||||
|
||||
echo "Built:"
|
||||
ls -1 "${DIST}"/*.rpm
|
||||
Reference in New Issue
Block a user