Add chlog CLI with chcat/chtail/chgrep entrypoints
Single Go binary for the ClickHouse log store (logs.raw): chlog with cat/tail/grep subcommands, plus chcat/chtail/chgrep argv[0]-dispatched symlink entrypoints. Every query is time-bounded and fully parameterized; chgrep guards wide unfiltered scans. Ships nfpm RPM with completions and woodpecker PR/tag pipelines mirroring node-lookup.
This commit is contained in:
@@ -0,0 +1,8 @@
|
|||||||
|
# built binaries (repo root only)
|
||||||
|
/chlog
|
||||||
|
/chcat
|
||||||
|
/chtail
|
||||||
|
/chgrep
|
||||||
|
# cross-compiled release artifacts (e.g. chlog-linux-amd64)
|
||||||
|
/chlog-*
|
||||||
|
dist/
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
repos:
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.6.0
|
||||||
|
hooks:
|
||||||
|
- id: trailing-whitespace
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
- id: check-yaml
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: mixed-line-ending
|
||||||
|
args: [--fix=lf]
|
||||||
|
|
||||||
|
- repo: https://github.com/dnephin/pre-commit-golang
|
||||||
|
rev: v0.5.1
|
||||||
|
hooks:
|
||||||
|
- id: go-fmt
|
||||||
|
- id: go-vet
|
||||||
|
- id: go-unit-tests
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: build
|
||||||
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- make build
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: pre-commit
|
||||||
|
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||||
|
commands:
|
||||||
|
- uvx pre-commit run --all-files
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
when:
|
||||||
|
- event: tag
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: test
|
||||||
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- go test -race ./...
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
|
|
||||||
|
# Build the single chlog binary into dist/ (consumed by the RPM step) plus
|
||||||
|
# cross-platform binaries attached to the Gitea release.
|
||||||
|
- name: build
|
||||||
|
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||||
|
commands:
|
||||||
|
- make build VERSION=${CI_COMMIT_TAG}
|
||||||
|
# $$ escapes shell vars so Woodpecker leaves them for the shell;
|
||||||
|
# ${CI_COMMIT_TAG} is a real Woodpecker var and stays single-$.
|
||||||
|
- |
|
||||||
|
for osarch in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do
|
||||||
|
os="$${osarch%/*}"; arch="$${osarch#*/}"
|
||||||
|
GOOS="$$os" GOARCH="$$arch" \
|
||||||
|
go build -ldflags="-s -w -X main.version=${CI_COMMIT_TAG}" \
|
||||||
|
-o "chlog-$${os}-$${arch}" .
|
||||||
|
done
|
||||||
|
depends_on: [test]
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
|
|
||||||
|
# Package the built binary + symlinks + shell completions into an RPM.
|
||||||
|
- name: package
|
||||||
|
image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest
|
||||||
|
commands:
|
||||||
|
- ./scripts/build-rpm.sh ${CI_COMMIT_TAG}
|
||||||
|
depends_on: [build]
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
|
|
||||||
|
# Publish the RPM to the artifactapi local rpm repo (a real yum repo;
|
||||||
|
# repodata regenerates automatically).
|
||||||
|
- name: upload-rpm
|
||||||
|
image: git.unkin.net/unkin/almalinux9-base:20260606
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
HOST="https://artifactapi.k8s.syd1.au.unkin.net"
|
||||||
|
REPO="rpm-internal"
|
||||||
|
for rpm in dist/*.rpm; do
|
||||||
|
FILE=$$(basename "$$rpm")
|
||||||
|
# artifactapi has no HEAD route (returns 405); probe with GET against
|
||||||
|
# the served path (RPMs are stored under Packages/) to avoid re-upload.
|
||||||
|
code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true)
|
||||||
|
if [ "$$code" = "200" ]; then
|
||||||
|
echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "Uploading $$FILE to $$REPO (existence probe returned $$code)"
|
||||||
|
curl -f -X PUT \
|
||||||
|
"$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \
|
||||||
|
-H "Content-Type: application/x-rpm" \
|
||||||
|
--data-binary @"$$rpm"
|
||||||
|
done
|
||||||
|
depends_on: [package]
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 128Mi
|
||||||
|
cpu: 100m
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 500m
|
||||||
|
|
||||||
|
# Cut a Gitea release with the cross-platform binaries attached.
|
||||||
|
- name: release
|
||||||
|
image: git.unkin.net/unkin/almalinux9-base:20260606
|
||||||
|
environment:
|
||||||
|
RELEASER_TOKEN:
|
||||||
|
from_secret: RELEASER_TOKEN
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
curl --output /usr/local/bin/tea https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/gitea-dl/tea/0.12.0/tea-0.12.0-linux-amd64 && chmod +x /usr/local/bin/tea
|
||||||
|
tea logins add --name gitea --url https://git.unkin.net --token "$${RELEASER_TOKEN}" --no-version-check
|
||||||
|
# Several tags can point at the same commit, so skip tags on the current
|
||||||
|
# commit and pick the newest semver tag that is a real ancestor.
|
||||||
|
CUR_SHA=$$(git rev-list -n1 "${CI_COMMIT_TAG}")
|
||||||
|
PREV_TAG=""
|
||||||
|
for t in $$(git tag --sort=-v:refname); do
|
||||||
|
[ "$$t" = "${CI_COMMIT_TAG}" ] && continue
|
||||||
|
[ "$$(git rev-list -n1 "$$t")" = "$$CUR_SHA" ] && continue
|
||||||
|
if git merge-base --is-ancestor "$$t" "${CI_COMMIT_TAG}" 2>/dev/null; then
|
||||||
|
PREV_TAG="$$t"; break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -n "$$PREV_TAG" ]; then
|
||||||
|
NOTES=$$(git log "$${PREV_TAG}..${CI_COMMIT_TAG}" --pretty=format:"- %s")
|
||||||
|
else
|
||||||
|
NOTES=$$(git log --pretty=format:"- %s")
|
||||||
|
fi
|
||||||
|
tea releases create --tag "${CI_COMMIT_TAG}" --title "${CI_COMMIT_TAG}" --note "$${NOTES}" --login gitea --repo "${CI_REPO}"
|
||||||
|
RPM=$$(ls dist/*.rpm 2>/dev/null | head -1)
|
||||||
|
ASSETS="chlog-linux-amd64 chlog-linux-arm64 chlog-darwin-amd64 chlog-darwin-arm64"
|
||||||
|
[ -n "$$RPM" ] && ASSETS="$$ASSETS $$RPM"
|
||||||
|
sha256sum $$ASSETS > sha256sums.txt
|
||||||
|
tea releases assets create "${CI_COMMIT_TAG}" $$ASSETS sha256sums.txt \
|
||||||
|
--login gitea --repo "${CI_REPO}"
|
||||||
|
depends_on: [upload-rpm]
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 128Mi
|
||||||
|
cpu: 100m
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 500m
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: lint
|
||||||
|
image: golangci/golangci-lint:latest
|
||||||
|
commands:
|
||||||
|
- golangci-lint run ./...
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
|
|
||||||
|
- name: test
|
||||||
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- go test -v -race ./...
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 2Gi
|
||||||
|
cpu: 2
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
BINARY := chlog
|
||||||
|
# chcat/chtail/chgrep are argv[0]-dispatched symlinks to the single chlog binary.
|
||||||
|
LINKS := chcat chtail chgrep
|
||||||
|
DIST := dist
|
||||||
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||||
|
GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)"
|
||||||
|
OS ?= $(shell go env GOOS)
|
||||||
|
ARCH ?= $(shell go env GOARCH)
|
||||||
|
|
||||||
|
.PHONY: all build test lint fmt clean install completions rpm rpm-package patch minor major _tag
|
||||||
|
|
||||||
|
all: build
|
||||||
|
|
||||||
|
build:
|
||||||
|
CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GOFLAGS) -o $(DIST)/$(BINARY) .
|
||||||
|
@for l in $(LINKS); do ln -sf $(BINARY) $(DIST)/$$l; done
|
||||||
|
|
||||||
|
test:
|
||||||
|
go test -v -race ./...
|
||||||
|
|
||||||
|
lint:
|
||||||
|
golangci-lint run ./...
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
gofmt -w .
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf $(DIST)
|
||||||
|
|
||||||
|
install:
|
||||||
|
go install $(GOFLAGS) .
|
||||||
|
|
||||||
|
# Generate bash/zsh/fish completions for chlog and each symlink entrypoint.
|
||||||
|
completions: build
|
||||||
|
@mkdir -p $(DIST)/completions
|
||||||
|
@for b in $(BINARY) $(LINKS); do \
|
||||||
|
$(DIST)/$$b completion bash > $(DIST)/completions/$$b.bash; \
|
||||||
|
$(DIST)/$$b completion zsh > $(DIST)/completions/_$$b; \
|
||||||
|
$(DIST)/$$b completion fish > $(DIST)/completions/$$b.fish; \
|
||||||
|
done
|
||||||
|
|
||||||
|
rpm: build rpm-package
|
||||||
|
|
||||||
|
rpm-package:
|
||||||
|
./scripts/build-rpm.sh $(VERSION)
|
||||||
|
|
||||||
|
# Bump helpers — reads the latest semver tag and creates the next one.
|
||||||
|
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||||
|
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||||
|
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||||
|
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
|
||||||
|
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
|
||||||
|
|
||||||
|
patch:
|
||||||
|
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
minor:
|
||||||
|
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
major:
|
||||||
|
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
|
||||||
|
git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW
|
||||||
|
|
||||||
|
_tag:
|
||||||
|
git push origin $(TAG)
|
||||||
@@ -1,3 +1,65 @@
|
|||||||
# clickhouse-tools
|
# clickhouse-tools
|
||||||
|
|
||||||
CLI tools (chcat, chtail, chgrep) for searching, filtering and tailing logs in the ClickHouse log store
|
CLI tools for the ClickHouse log store (`logs.raw`): one binary, `chlog`, with
|
||||||
|
three entrypoints installed as symlinks:
|
||||||
|
|
||||||
|
| Command | Also as | Does |
|
||||||
|
|----------|---------------|------|
|
||||||
|
| `chcat` | `chlog cat` | Print logs oldest-first over a bounded time range |
|
||||||
|
| `chtail` | `chlog tail` | Follow logs live (2s poll, overlap + dedupe so nothing is lost or repeated) |
|
||||||
|
| `chgrep` | `chlog grep` | Search log messages (substring, `-i`, `--regex`) |
|
||||||
|
|
||||||
|
## Why time bounds everywhere
|
||||||
|
|
||||||
|
`logs.raw` has no text index and holds ~281M rows/day (3-day TTL). An unbounded
|
||||||
|
message scan takes ~1 minute and the server kills queries at 120s. Every query
|
||||||
|
these tools issue is therefore time-bounded — the default range is the last
|
||||||
|
hour (`--since 1h`) — and `chgrep` refuses a search wider than 6h with no
|
||||||
|
`--namespace`/`--host`/`--app` filter unless you pass `--force`.
|
||||||
|
|
||||||
|
All user input travels as ClickHouse HTTP `{name:Type}` parameters; nothing is
|
||||||
|
ever interpolated into SQL text.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```sh
|
||||||
|
chcat -n logging --since 30m
|
||||||
|
chcat --host web01 --since 2h --until 1h --format logfmt
|
||||||
|
chtail -n media --app jellyfin
|
||||||
|
chgrep -n kube-system -i "connection refused" --since 4h
|
||||||
|
chgrep --app vector --regex 'timed? ?out' --since 1d
|
||||||
|
chgrep --fields req_id=42 -n api "payment"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Common flags
|
||||||
|
|
||||||
|
- `--since` / `--until` — duration ago (`15m`, `1h`, `2d`, `1w`) or RFC3339;
|
||||||
|
default `--since 1h`, `--until` now
|
||||||
|
- `-n/--namespace`, `--host`, `--pod`, `--container`, `--app` (labels['app']),
|
||||||
|
`--severity` (case-insensitive), `--stream`, `--source`
|
||||||
|
- `--limit` — max rows (default 10000 for cat/grep; tail is unlimited)
|
||||||
|
- `--format text|json|logfmt` — text is `ts ns/pod msg` (host for vm rows),
|
||||||
|
colored only on a TTY (`NO_COLOR` respected)
|
||||||
|
|
||||||
|
### chgrep extras
|
||||||
|
|
||||||
|
- pattern is a substring by default; `-i` case-insensitive; `--regex` RE2 (`match()`)
|
||||||
|
- `--fields key=value` (repeatable) filters the structured `fields` map
|
||||||
|
- `--force` overrides the wide-unfiltered-search guard
|
||||||
|
|
||||||
|
## Connection
|
||||||
|
|
||||||
|
| Env | Default |
|
||||||
|
|-----|---------|
|
||||||
|
| `CH_URL` | `http://clickhouse-logs.logging.svc.cluster.local:8123` |
|
||||||
|
| `CH_USER` | `logreader` |
|
||||||
|
| `CH_PASSWORD` | (empty) |
|
||||||
|
|
||||||
|
## Build and release
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make build # dist/chlog + symlinks
|
||||||
|
make test # go test -race ./...
|
||||||
|
make rpm # nfpm RPM with binary, symlinks, bash/zsh/fish completions
|
||||||
|
make patch # tag + push next vX.Y.Z → CI releases RPM to artifactapi rpm-internal
|
||||||
|
```
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
module git.unkin.net/unkin/clickhouse-tools
|
||||||
|
|
||||||
|
go 1.25
|
||||||
|
|
||||||
|
require github.com/spf13/cobra v1.10.2
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"hash/fnv"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const chTimeLayout = "2006-01-02 15:04:05.999"
|
||||||
|
|
||||||
|
type Row struct {
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
Host string `json:"host"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
Pod string `json:"pod"`
|
||||||
|
Container string `json:"container"`
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Severity string `json:"severity"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Labels map[string]string `json:"labels"`
|
||||||
|
Fields map[string]string `json:"fields"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r Row) Time() time.Time {
|
||||||
|
t, err := time.Parse(chTimeLayout, r.Timestamp)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return t.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key identifies a row for overlap dedupe during paging and tailing.
|
||||||
|
func (r Row) Key() uint64 {
|
||||||
|
h := fnv.New64a()
|
||||||
|
for _, s := range []string{r.Timestamp, r.Host, r.Source, r.Namespace, r.Pod, r.Container, r.Stream, r.Message} {
|
||||||
|
io.WriteString(h, s)
|
||||||
|
h.Write([]byte{0})
|
||||||
|
}
|
||||||
|
return h.Sum64()
|
||||||
|
}
|
||||||
|
|
||||||
|
type Client struct {
|
||||||
|
cfg Config
|
||||||
|
http *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewClient(cfg Config) *Client {
|
||||||
|
return &Client{cfg: cfg, http: &http.Client{Timeout: 130 * time.Second}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes the query and streams each result row to fn. The SQL travels
|
||||||
|
// in the request body; every value goes as a param_* HTTP parameter.
|
||||||
|
func (c *Client) Run(ctx context.Context, q Query, fn func(Row) error) error {
|
||||||
|
v := url.Values{}
|
||||||
|
v.Set("default_format", "JSONEachRow")
|
||||||
|
for name, value := range q.Params {
|
||||||
|
v.Set("param_"+name, value)
|
||||||
|
}
|
||||||
|
u := strings.TrimRight(c.cfg.URL, "/") + "/?" + v.Encode()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(q.SQL))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "text/plain")
|
||||||
|
req.Header.Set("X-ClickHouse-User", c.cfg.User)
|
||||||
|
if c.cfg.Password != "" {
|
||||||
|
req.Header.Set("X-ClickHouse-Key", c.cfg.Password)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clickhouse request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
|
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||||
|
}
|
||||||
|
|
||||||
|
sc := bufio.NewScanner(resp.Body)
|
||||||
|
sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||||
|
line := 0
|
||||||
|
for sc.Scan() {
|
||||||
|
line++
|
||||||
|
b := sc.Bytes()
|
||||||
|
if len(b) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var r Row
|
||||||
|
if err := json.Unmarshal(b, &r); err != nil {
|
||||||
|
return fmt.Errorf("parse result row %s: %w", strconv.Itoa(line), err)
|
||||||
|
}
|
||||||
|
if err := fn(r); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sc.Err()
|
||||||
|
}
|
||||||
@@ -0,0 +1,279 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func chTS(t time.Time) string {
|
||||||
|
return t.UTC().Format("2006-01-02 15:04:05.000")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mkRow(ts time.Time, msg string) Row {
|
||||||
|
return Row{
|
||||||
|
Timestamp: chTS(ts),
|
||||||
|
Host: "node1",
|
||||||
|
Source: "k8s",
|
||||||
|
Namespace: "logging",
|
||||||
|
Pod: "vector-abc",
|
||||||
|
Container: "vector",
|
||||||
|
Stream: "stdout",
|
||||||
|
Message: msg,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeCH serves the ClickHouse HTTP contract used by Client: JSONEachRow rows
|
||||||
|
// from an in-memory store, honoring the since/until/limit query parameters.
|
||||||
|
type fakeCH struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
rows []Row
|
||||||
|
queries int
|
||||||
|
lastUser string
|
||||||
|
lastPass string
|
||||||
|
lastSQL string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCH) add(rows ...Row) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.rows = append(f.rows, rows...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCH) handler(t *testing.T) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.queries++
|
||||||
|
f.lastUser = r.Header.Get("X-ClickHouse-User")
|
||||||
|
f.lastPass = r.Header.Get("X-ClickHouse-Key")
|
||||||
|
body := make([]byte, r.ContentLength)
|
||||||
|
r.Body.Read(body)
|
||||||
|
f.lastSQL = string(body)
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
since, err1 := strconv.ParseInt(q.Get("param_since_ms"), 10, 64)
|
||||||
|
until, err2 := strconv.ParseInt(q.Get("param_until_ms"), 10, 64)
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
t.Errorf("query missing time bound params: %s", r.URL.RawQuery)
|
||||||
|
http.Error(w, "unbounded query", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var limit int64 = -1
|
||||||
|
if s := q.Get("param_limit"); s != "" {
|
||||||
|
limit, _ = strconv.ParseInt(s, 10, 64)
|
||||||
|
}
|
||||||
|
var sent int64
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
for _, row := range f.rows {
|
||||||
|
ms := row.Time().UnixMilli()
|
||||||
|
if ms < since || ms >= until {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if limit >= 0 && sent >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
enc.Encode(row)
|
||||||
|
sent++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientRun(t *testing.T) {
|
||||||
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
fake := &fakeCH{}
|
||||||
|
fake.add(mkRow(base, "hello"), mkRow(base.Add(time.Second), "world"))
|
||||||
|
srv := httptest.NewServer(fake.handler(t))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader", Password: "secret"})
|
||||||
|
q, err := Build(Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Minute)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got []Row
|
||||||
|
if err := c.Run(context.Background(), q, func(r Row) error {
|
||||||
|
got = append(got, r)
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 2 || got[0].Message != "hello" || got[1].Message != "world" {
|
||||||
|
t.Fatalf("rows = %+v", got)
|
||||||
|
}
|
||||||
|
if fake.lastUser != "logreader" || fake.lastPass != "secret" {
|
||||||
|
t.Errorf("auth headers: user=%q pass set=%v", fake.lastUser, fake.lastPass != "")
|
||||||
|
}
|
||||||
|
if !strings.Contains(fake.lastSQL, "FROM logs.raw") {
|
||||||
|
t.Errorf("SQL body not sent: %q", fake.lastSQL)
|
||||||
|
}
|
||||||
|
if got[0].Time().IsZero() {
|
||||||
|
t.Error("timestamp did not parse")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientRunHTTPError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "Code: 159. DB::Exception: Timeout exceeded", http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
||||||
|
q, _ := Build(Filter{Since: time.Now().Add(-time.Hour), Until: time.Now()})
|
||||||
|
err := c.Run(context.Background(), q, func(Row) error { return nil })
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "HTTP 500") || !strings.Contains(err.Error(), "Timeout exceeded") {
|
||||||
|
t.Fatalf("err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPagePagesAndDedupes(t *testing.T) {
|
||||||
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
fake := &fakeCH{}
|
||||||
|
// 25 rows, including three rows sharing one boundary timestamp.
|
||||||
|
for i := 0; i < 25; i++ {
|
||||||
|
ts := base.Add(time.Duration(i) * time.Second)
|
||||||
|
if i >= 10 && i < 13 {
|
||||||
|
ts = base.Add(10 * time.Second)
|
||||||
|
}
|
||||||
|
fake.add(mkRow(ts, fmt.Sprintf("msg-%d", i)))
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(fake.handler(t))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
||||||
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour)}
|
||||||
|
var got []string
|
||||||
|
n, err := Page(context.Background(), c, f, 7, func(r Row) error {
|
||||||
|
got = append(got, r.Message)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 25 || len(got) != 25 {
|
||||||
|
t.Fatalf("emitted %d rows (%d reported), want 25: %v", len(got), n, got)
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, m := range got {
|
||||||
|
if seen[m] {
|
||||||
|
t.Fatalf("duplicate row %q emitted", m)
|
||||||
|
}
|
||||||
|
seen[m] = true
|
||||||
|
}
|
||||||
|
if fake.queries < 4 {
|
||||||
|
t.Errorf("expected multiple paged queries, got %d", fake.queries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPageHonorsLimit(t *testing.T) {
|
||||||
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
fake := &fakeCH{}
|
||||||
|
for i := 0; i < 30; i++ {
|
||||||
|
fake.add(mkRow(base.Add(time.Duration(i)*time.Second), fmt.Sprintf("msg-%d", i)))
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(fake.handler(t))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
||||||
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour), Limit: 12}
|
||||||
|
var got []string
|
||||||
|
n, err := Page(context.Background(), c, f, 5, func(r Row) error {
|
||||||
|
got = append(got, r.Message)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 12 || len(got) != 12 || got[0] != "msg-0" || got[11] != "msg-11" {
|
||||||
|
t.Fatalf("emitted %d rows: %v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPageSingleMillisecondBurst(t *testing.T) {
|
||||||
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
fake := &fakeCH{}
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
fake.add(mkRow(base, fmt.Sprintf("burst-%d", i)))
|
||||||
|
}
|
||||||
|
fake.add(mkRow(base.Add(time.Second), "after"))
|
||||||
|
srv := httptest.NewServer(fake.handler(t))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
||||||
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour)}
|
||||||
|
var got []string
|
||||||
|
_, err := Page(context.Background(), c, f, 3, func(r Row) error {
|
||||||
|
got = append(got, r.Message)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 7 || got[6] != "after" {
|
||||||
|
t.Fatalf("rows = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTailDedupesAcrossPolls(t *testing.T) {
|
||||||
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
fake := &fakeCH{}
|
||||||
|
fake.add(mkRow(base, "initial-1"), mkRow(base.Add(time.Second), "initial-2"))
|
||||||
|
srv := httptest.NewServer(fake.handler(t))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
||||||
|
|
||||||
|
var mu sync.Mutex
|
||||||
|
var got []string
|
||||||
|
polls := 0
|
||||||
|
now := func() time.Time {
|
||||||
|
return base.Add(time.Duration(10+polls) * time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
emit := func(r Row) error {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
got = append(got, r.Message)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- Tail(ctx, c, Filter{Since: base.Add(-time.Minute)}, func() time.Time {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
polls++
|
||||||
|
return now()
|
||||||
|
}, 10*time.Millisecond, emit)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Late-arriving row inside the overlap window plus a genuinely new row.
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
fake.add(mkRow(base.Add(500*time.Millisecond), "late"), mkRow(base.Add(2*time.Second), "new"))
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err != context.Canceled {
|
||||||
|
t.Fatalf("tail err = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
counts := map[string]int{}
|
||||||
|
for _, m := range got {
|
||||||
|
counts[m]++
|
||||||
|
}
|
||||||
|
for _, want := range []string{"initial-1", "initial-2", "late", "new"} {
|
||||||
|
if counts[want] != 1 {
|
||||||
|
t.Errorf("row %q emitted %d times; all: %v", want, counts[want], got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import "os"
|
||||||
|
|
||||||
|
const (
|
||||||
|
DefaultURL = "http://clickhouse-logs.logging.svc.cluster.local:8123"
|
||||||
|
DefaultUser = "logreader"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
URL string
|
||||||
|
User string
|
||||||
|
Password string
|
||||||
|
}
|
||||||
|
|
||||||
|
func ConfigFromEnv() Config {
|
||||||
|
cfg := Config{
|
||||||
|
URL: os.Getenv("CH_URL"),
|
||||||
|
User: os.Getenv("CH_USER"),
|
||||||
|
Password: os.Getenv("CH_PASSWORD"),
|
||||||
|
}
|
||||||
|
if cfg.URL == "" {
|
||||||
|
cfg.URL = DefaultURL
|
||||||
|
}
|
||||||
|
if cfg.User == "" {
|
||||||
|
cfg.User = DefaultUser
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
colReset = "\x1b[0m"
|
||||||
|
colDim = "\x1b[2m"
|
||||||
|
colCyan = "\x1b[36m"
|
||||||
|
colGreen = "\x1b[32m"
|
||||||
|
colRed = "\x1b[31m"
|
||||||
|
colYel = "\x1b[33m"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Formatter func(w io.Writer, r Row) error
|
||||||
|
|
||||||
|
func NewFormatter(format string, color bool) (Formatter, error) {
|
||||||
|
switch format {
|
||||||
|
case "text":
|
||||||
|
return textFormatter(color), nil
|
||||||
|
case "json":
|
||||||
|
return jsonFormatter, nil
|
||||||
|
case "logfmt":
|
||||||
|
return logfmtFormatter, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown format %q: want text, json or logfmt", format)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// origin renders the row's source identity: ns/pod for k8s rows, host for vm.
|
||||||
|
func origin(r Row) string {
|
||||||
|
if r.Pod != "" {
|
||||||
|
return r.Namespace + "/" + r.Pod
|
||||||
|
}
|
||||||
|
if r.Namespace != "" {
|
||||||
|
return r.Namespace
|
||||||
|
}
|
||||||
|
return r.Host
|
||||||
|
}
|
||||||
|
|
||||||
|
func severityColor(sev string) string {
|
||||||
|
switch strings.ToLower(sev) {
|
||||||
|
case "error", "err", "fatal", "critical", "crit":
|
||||||
|
return colRed
|
||||||
|
case "warn", "warning":
|
||||||
|
return colYel
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func textFormatter(color bool) Formatter {
|
||||||
|
return func(w io.Writer, r Row) error {
|
||||||
|
ts := r.Time().Format("2006-01-02T15:04:05.000Z")
|
||||||
|
msg := r.Message
|
||||||
|
if !color {
|
||||||
|
_, err := fmt.Fprintf(w, "%s %s %s\n", ts, origin(r), msg)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c := severityColor(r.Severity); c != "" {
|
||||||
|
msg = c + msg + colReset
|
||||||
|
}
|
||||||
|
_, err := fmt.Fprintf(w, "%s%s%s %s%s%s %s\n",
|
||||||
|
colDim, ts, colReset, colCyan, origin(r), colReset, msg)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type jsonRow struct {
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
Host string `json:"host"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Namespace string `json:"namespace,omitempty"`
|
||||||
|
Pod string `json:"pod,omitempty"`
|
||||||
|
Container string `json:"container,omitempty"`
|
||||||
|
Stream string `json:"stream,omitempty"`
|
||||||
|
Severity string `json:"severity,omitempty"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Labels map[string]string `json:"labels,omitempty"`
|
||||||
|
Fields map[string]string `json:"fields,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonFormatter(w io.Writer, r Row) error {
|
||||||
|
b, err := json.Marshal(jsonRow{
|
||||||
|
Timestamp: r.Time().Format(time.RFC3339Nano),
|
||||||
|
Host: r.Host,
|
||||||
|
Source: r.Source,
|
||||||
|
Namespace: r.Namespace,
|
||||||
|
Pod: r.Pod,
|
||||||
|
Container: r.Container,
|
||||||
|
Stream: r.Stream,
|
||||||
|
Severity: r.Severity,
|
||||||
|
Message: r.Message,
|
||||||
|
Labels: r.Labels,
|
||||||
|
Fields: r.Fields,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(w, "%s\n", b)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func logfmtValue(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return `""`
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(s, " \t\"=\n") {
|
||||||
|
return fmt.Sprintf("%q", s)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func logfmtFormatter(w io.Writer, r Row) error {
|
||||||
|
var b strings.Builder
|
||||||
|
pair := func(k, v string) {
|
||||||
|
if v == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
b.WriteString(k)
|
||||||
|
b.WriteByte('=')
|
||||||
|
b.WriteString(logfmtValue(v))
|
||||||
|
}
|
||||||
|
pair("ts", r.Time().Format(time.RFC3339Nano))
|
||||||
|
pair("source", r.Source)
|
||||||
|
pair("host", r.Host)
|
||||||
|
pair("ns", r.Namespace)
|
||||||
|
pair("pod", r.Pod)
|
||||||
|
pair("container", r.Container)
|
||||||
|
pair("stream", r.Stream)
|
||||||
|
pair("severity", r.Severity)
|
||||||
|
pair("app", r.Labels["app"])
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
b.WriteString("msg=")
|
||||||
|
b.WriteString(logfmtValue(r.Message))
|
||||||
|
b.WriteByte('\n')
|
||||||
|
_, err := io.WriteString(w, b.String())
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sampleRow() Row {
|
||||||
|
return Row{
|
||||||
|
Timestamp: "2026-08-23 05:00:01.234",
|
||||||
|
Host: "node1",
|
||||||
|
Source: "k8s",
|
||||||
|
Namespace: "logging",
|
||||||
|
Pod: "vector-abc",
|
||||||
|
Container: "vector",
|
||||||
|
Stream: "stdout",
|
||||||
|
Severity: "Error",
|
||||||
|
Message: "something broke",
|
||||||
|
Labels: map[string]string{"app": "vector"},
|
||||||
|
Fields: map[string]string{"req_id": "42"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func render(t *testing.T, format string, color bool, r Row) string {
|
||||||
|
t.Helper()
|
||||||
|
f, err := NewFormatter(format, color)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
if err := f(&b, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextNoColor(t *testing.T) {
|
||||||
|
out := render(t, "text", false, sampleRow())
|
||||||
|
want := "2026-08-23T05:00:01.234Z logging/vector-abc something broke\n"
|
||||||
|
if out != want {
|
||||||
|
t.Errorf("got %q, want %q", out, want)
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "\x1b[") {
|
||||||
|
t.Error("no-color output contains ANSI escapes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextColor(t *testing.T) {
|
||||||
|
out := render(t, "text", true, sampleRow())
|
||||||
|
if !strings.Contains(out, "\x1b[") {
|
||||||
|
t.Error("color output missing ANSI escapes")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "logging/vector-abc") || !strings.Contains(out, "something broke") {
|
||||||
|
t.Errorf("content missing: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextVMRowUsesHost(t *testing.T) {
|
||||||
|
r := sampleRow()
|
||||||
|
r.Source = "vm"
|
||||||
|
r.Namespace = ""
|
||||||
|
r.Pod = ""
|
||||||
|
out := render(t, "text", false, r)
|
||||||
|
if !strings.Contains(out, " node1 ") {
|
||||||
|
t.Errorf("vm row should show host: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJSONFormat(t *testing.T) {
|
||||||
|
out := render(t, "json", false, sampleRow())
|
||||||
|
var m map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(out), &m); err != nil {
|
||||||
|
t.Fatalf("not valid JSON: %v: %q", err, out)
|
||||||
|
}
|
||||||
|
if m["timestamp"] != "2026-08-23T05:00:01.234Z" {
|
||||||
|
t.Errorf("timestamp = %v", m["timestamp"])
|
||||||
|
}
|
||||||
|
if m["message"] != "something broke" || m["namespace"] != "logging" {
|
||||||
|
t.Errorf("fields wrong: %v", m)
|
||||||
|
}
|
||||||
|
labels, _ := m["labels"].(map[string]any)
|
||||||
|
if labels["app"] != "vector" {
|
||||||
|
t.Errorf("labels = %v", m["labels"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogfmtFormat(t *testing.T) {
|
||||||
|
out := render(t, "logfmt", false, sampleRow())
|
||||||
|
for _, want := range []string{
|
||||||
|
"ts=2026-08-23T05:00:01.234Z",
|
||||||
|
"ns=logging",
|
||||||
|
"pod=vector-abc",
|
||||||
|
"severity=Error",
|
||||||
|
"app=vector",
|
||||||
|
`msg="something broke"`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Errorf("logfmt missing %q: %q", want, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(out, "\n") {
|
||||||
|
t.Error("missing trailing newline")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogfmtQuoting(t *testing.T) {
|
||||||
|
r := sampleRow()
|
||||||
|
r.Message = `plain`
|
||||||
|
out := render(t, "logfmt", false, r)
|
||||||
|
if !strings.Contains(out, "msg=plain") {
|
||||||
|
t.Errorf("unquoted simple value expected: %q", out)
|
||||||
|
}
|
||||||
|
r.Message = "has \"quotes\" and = signs"
|
||||||
|
out = render(t, "logfmt", false, r)
|
||||||
|
if !strings.Contains(out, `msg="has \"quotes\" and = signs"`) {
|
||||||
|
t.Errorf("quoted value wrong: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownFormat(t *testing.T) {
|
||||||
|
if _, err := NewFormatter("yaml", false); err == nil {
|
||||||
|
t.Fatal("expected error for unknown format")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRowKeyStableAndDistinct(t *testing.T) {
|
||||||
|
a := sampleRow()
|
||||||
|
b := sampleRow()
|
||||||
|
if a.Key() != b.Key() {
|
||||||
|
t.Error("identical rows should share a key")
|
||||||
|
}
|
||||||
|
b.Message = "different"
|
||||||
|
if a.Key() == b.Key() {
|
||||||
|
t.Error("different rows should not share a key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRowTimeParses(t *testing.T) {
|
||||||
|
r := sampleRow()
|
||||||
|
want := time.Date(2026, 8, 23, 5, 0, 1, 234000000, time.UTC)
|
||||||
|
if !r.Time().Equal(want) {
|
||||||
|
t.Errorf("Time() = %s, want %s", r.Time(), want)
|
||||||
|
}
|
||||||
|
r.Timestamp = "2026-08-23 05:00:01"
|
||||||
|
if r.Time().IsZero() {
|
||||||
|
t.Error("timestamp without fraction should still parse")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const DefaultPageSize = 10000
|
||||||
|
|
||||||
|
type runner interface {
|
||||||
|
Run(ctx context.Context, q Query, fn func(Row) error) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Page walks the filter's time range as a series of bounded keyset-paged
|
||||||
|
// queries: each page re-queries from the last-seen timestamp (inclusive, so
|
||||||
|
// nothing on the boundary millisecond is lost) and dedupes the overlap. Each
|
||||||
|
// page requests len(seen) extra rows on top of the wanted count, so known
|
||||||
|
// boundary duplicates can never starve progress. Returns rows emitted.
|
||||||
|
func Page(ctx context.Context, c runner, f Filter, pageSize uint64, emit func(Row) error) (uint64, error) {
|
||||||
|
if pageSize == 0 {
|
||||||
|
pageSize = DefaultPageSize
|
||||||
|
}
|
||||||
|
budget := f.Limit
|
||||||
|
|
||||||
|
cursor := f.Since
|
||||||
|
seen := map[uint64]struct{}{}
|
||||||
|
var emitted uint64
|
||||||
|
|
||||||
|
for {
|
||||||
|
want := pageSize
|
||||||
|
if budget > 0 && budget-emitted < want {
|
||||||
|
want = budget - emitted
|
||||||
|
}
|
||||||
|
pf := f
|
||||||
|
pf.Since = cursor
|
||||||
|
pf.Limit = want + uint64(len(seen))
|
||||||
|
q, err := Build(pf)
|
||||||
|
if err != nil {
|
||||||
|
return emitted, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var got uint64
|
||||||
|
var lastTS time.Time
|
||||||
|
pageSeen := map[uint64]struct{}{}
|
||||||
|
err = c.Run(ctx, q, func(r Row) error {
|
||||||
|
got++
|
||||||
|
if budget > 0 && emitted >= budget {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ts := r.Time()
|
||||||
|
k := r.Key()
|
||||||
|
if ts.Equal(cursor) {
|
||||||
|
if _, dup := seen[k]; dup {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen[k] = struct{}{}
|
||||||
|
} else {
|
||||||
|
if !ts.Equal(lastTS) {
|
||||||
|
lastTS = ts
|
||||||
|
pageSeen = map[uint64]struct{}{}
|
||||||
|
}
|
||||||
|
pageSeen[k] = struct{}{}
|
||||||
|
}
|
||||||
|
emitted++
|
||||||
|
return emit(r)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return emitted, err
|
||||||
|
}
|
||||||
|
if got < pf.Limit || (budget > 0 && emitted >= budget) {
|
||||||
|
return emitted, nil
|
||||||
|
}
|
||||||
|
if lastTS.After(cursor) {
|
||||||
|
cursor = lastTS
|
||||||
|
seen = pageSeen
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Table = "logs.raw"
|
||||||
|
|
||||||
|
// Filter describes one bounded query against logs.raw. Since/Until are
|
||||||
|
// mandatory: the table has no text index, so every query must be time-bounded.
|
||||||
|
type Filter struct {
|
||||||
|
Since time.Time
|
||||||
|
Until time.Time
|
||||||
|
|
||||||
|
Namespace string
|
||||||
|
Host string
|
||||||
|
Pod string
|
||||||
|
Container string
|
||||||
|
App string
|
||||||
|
Severity string
|
||||||
|
Stream string
|
||||||
|
Source string
|
||||||
|
|
||||||
|
Pattern string
|
||||||
|
Regex bool
|
||||||
|
IgnoreCase bool
|
||||||
|
Fields map[string]string
|
||||||
|
|
||||||
|
Limit uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Selective reports whether the filter narrows the scan enough to be cheap:
|
||||||
|
// any of namespace, host, or app restricts to a small slice of the table.
|
||||||
|
func (f Filter) Selective() bool {
|
||||||
|
return f.Namespace != "" || f.Host != "" || f.App != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type Query struct {
|
||||||
|
SQL string
|
||||||
|
Params map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
const selectColumns = "timestamp, host, source, namespace, pod, container, stream, severity, message, labels, fields"
|
||||||
|
|
||||||
|
func chTime(t time.Time) string {
|
||||||
|
return strconv.FormatInt(t.UnixMilli(), 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build renders a fully parameterized query. All user-supplied values travel
|
||||||
|
// as HTTP {name:Type} parameters, never interpolated into the SQL text.
|
||||||
|
func Build(f Filter) (Query, error) {
|
||||||
|
if f.Since.IsZero() || f.Until.IsZero() {
|
||||||
|
return Query{}, fmt.Errorf("query must be time-bounded: since/until missing")
|
||||||
|
}
|
||||||
|
if !f.Since.Before(f.Until) {
|
||||||
|
return Query{}, fmt.Errorf("empty time range: since %s is not before until %s",
|
||||||
|
f.Since.UTC().Format(time.RFC3339), f.Until.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|
||||||
|
params := map[string]string{
|
||||||
|
"since_ms": chTime(f.Since),
|
||||||
|
"until_ms": chTime(f.Until),
|
||||||
|
}
|
||||||
|
where := []string{
|
||||||
|
"timestamp >= fromUnixTimestamp64Milli({since_ms:Int64})",
|
||||||
|
"timestamp < fromUnixTimestamp64Milli({until_ms:Int64})",
|
||||||
|
}
|
||||||
|
|
||||||
|
addEq := func(column, name, value string) {
|
||||||
|
if value == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
where = append(where, fmt.Sprintf("%s = {%s:String}", column, name))
|
||||||
|
params[name] = value
|
||||||
|
}
|
||||||
|
addEq("namespace", "ns", f.Namespace)
|
||||||
|
addEq("host", "host", f.Host)
|
||||||
|
addEq("pod", "pod", f.Pod)
|
||||||
|
addEq("container", "container", f.Container)
|
||||||
|
addEq("stream", "stream", f.Stream)
|
||||||
|
addEq("source", "source", f.Source)
|
||||||
|
addEq("labels['app']", "app", f.App)
|
||||||
|
|
||||||
|
if f.Severity != "" {
|
||||||
|
where = append(where, "lowerUTF8(severity) = {severity:String}")
|
||||||
|
params["severity"] = strings.ToLower(f.Severity)
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.Pattern != "" {
|
||||||
|
switch {
|
||||||
|
case f.Regex:
|
||||||
|
pat := f.Pattern
|
||||||
|
if f.IgnoreCase {
|
||||||
|
pat = "(?i)" + pat
|
||||||
|
}
|
||||||
|
where = append(where, "match(message, {pattern:String})")
|
||||||
|
params["pattern"] = pat
|
||||||
|
case f.IgnoreCase:
|
||||||
|
where = append(where, "positionCaseInsensitive(message, {pattern:String}) > 0")
|
||||||
|
params["pattern"] = f.Pattern
|
||||||
|
default:
|
||||||
|
where = append(where, "position(message, {pattern:String}) > 0")
|
||||||
|
params["pattern"] = f.Pattern
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(f.Fields))
|
||||||
|
for k := range f.Fields {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for i, k := range keys {
|
||||||
|
kn := fmt.Sprintf("fk%d", i)
|
||||||
|
vn := fmt.Sprintf("fv%d", i)
|
||||||
|
where = append(where, fmt.Sprintf("fields[{%s:String}] = {%s:String}", kn, vn))
|
||||||
|
params[kn] = k
|
||||||
|
params[vn] = f.Fields[k]
|
||||||
|
}
|
||||||
|
|
||||||
|
sql := fmt.Sprintf("SELECT %s FROM %s WHERE %s ORDER BY timestamp ASC",
|
||||||
|
selectColumns, Table, strings.Join(where, " AND "))
|
||||||
|
if f.Limit > 0 {
|
||||||
|
sql += " LIMIT {limit:UInt64}"
|
||||||
|
params["limit"] = strconv.FormatUint(f.Limit, 10)
|
||||||
|
}
|
||||||
|
return Query{SQL: sql, Params: params}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
tSince = time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
||||||
|
tUntil = time.Date(2026, 8, 23, 6, 0, 0, 0, time.UTC)
|
||||||
|
)
|
||||||
|
|
||||||
|
func baseFilter() Filter {
|
||||||
|
return Filter{Since: tSince, Until: tUntil}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRequiresBounds(t *testing.T) {
|
||||||
|
if _, err := Build(Filter{Until: tUntil}); err == nil {
|
||||||
|
t.Fatal("expected error when since missing")
|
||||||
|
}
|
||||||
|
if _, err := Build(Filter{Since: tSince}); err == nil {
|
||||||
|
t.Fatal("expected error when until missing")
|
||||||
|
}
|
||||||
|
if _, err := Build(Filter{Since: tUntil, Until: tSince}); err == nil {
|
||||||
|
t.Fatal("expected error when since >= until")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildAlwaysTimeBounded(t *testing.T) {
|
||||||
|
q, err := Build(baseFilter())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"timestamp >= fromUnixTimestamp64Milli({since_ms:Int64})",
|
||||||
|
"timestamp < fromUnixTimestamp64Milli({until_ms:Int64})",
|
||||||
|
"ORDER BY timestamp ASC",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(q.SQL, want) {
|
||||||
|
t.Errorf("SQL missing %q:\n%s", want, q.SQL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if q.Params["since_ms"] != strconv.FormatInt(tSince.UnixMilli(), 10) {
|
||||||
|
t.Errorf("since_ms = %s", q.Params["since_ms"])
|
||||||
|
}
|
||||||
|
if q.Params["until_ms"] != strconv.FormatInt(tUntil.UnixMilli(), 10) {
|
||||||
|
t.Errorf("until_ms = %s", q.Params["until_ms"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildNoUserInputInSQL(t *testing.T) {
|
||||||
|
f := baseFilter()
|
||||||
|
f.Namespace = "evil'; DROP TABLE logs.raw; --"
|
||||||
|
f.Pattern = "inject{p:String}"
|
||||||
|
f.App = "x' OR 1=1"
|
||||||
|
f.Fields = map[string]string{"k'": "v\""}
|
||||||
|
q, err := Build(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, needle := range []string{f.Namespace, f.Pattern, f.App, "k'", "v\"", "DROP"} {
|
||||||
|
if strings.Contains(q.SQL, needle) {
|
||||||
|
t.Errorf("user input %q leaked into SQL:\n%s", needle, q.SQL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if q.Params["ns"] != f.Namespace || q.Params["pattern"] != f.Pattern {
|
||||||
|
t.Errorf("params missing user values: %v", q.Params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildFilters(t *testing.T) {
|
||||||
|
f := baseFilter()
|
||||||
|
f.Namespace = "logging"
|
||||||
|
f.Host = "node1"
|
||||||
|
f.Pod = "vector-abc"
|
||||||
|
f.Container = "vector"
|
||||||
|
f.Stream = "stderr"
|
||||||
|
f.Source = "k8s"
|
||||||
|
f.App = "vector"
|
||||||
|
f.Severity = "ERROR"
|
||||||
|
f.Limit = 100
|
||||||
|
q, err := Build(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for clause, param := range map[string]string{
|
||||||
|
"namespace = {ns:String}": "ns",
|
||||||
|
"host = {host:String}": "host",
|
||||||
|
"pod = {pod:String}": "pod",
|
||||||
|
"container = {container:String}": "container",
|
||||||
|
"stream = {stream:String}": "stream",
|
||||||
|
"source = {source:String}": "source",
|
||||||
|
"labels['app'] = {app:String}": "app",
|
||||||
|
"lowerUTF8(severity) = {severity:String}": "severity",
|
||||||
|
"LIMIT {limit:UInt64}": "limit",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(q.SQL, clause) {
|
||||||
|
t.Errorf("SQL missing %q", clause)
|
||||||
|
}
|
||||||
|
if _, ok := q.Params[param]; !ok {
|
||||||
|
t.Errorf("param %q missing", param)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if q.Params["severity"] != "error" {
|
||||||
|
t.Errorf("severity not lowercased: %q", q.Params["severity"])
|
||||||
|
}
|
||||||
|
if q.Params["limit"] != "100" {
|
||||||
|
t.Errorf("limit = %q", q.Params["limit"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildEmptyFiltersOmitted(t *testing.T) {
|
||||||
|
q, err := Build(baseFilter())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, where, ok := strings.Cut(q.SQL, " WHERE ")
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("no WHERE clause:\n%s", q.SQL)
|
||||||
|
}
|
||||||
|
for _, clause := range []string{"namespace =", "host =", "pod =", "labels[", "lowerUTF8", "position", "match", "LIMIT"} {
|
||||||
|
if strings.Contains(where, clause) {
|
||||||
|
t.Errorf("unexpected clause %q in WHERE:\n%s", clause, where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(q.Params) != 2 {
|
||||||
|
t.Errorf("want only time params, got %v", q.Params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildGrepVariants(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
regex, ignoreCase bool
|
||||||
|
wantClause string
|
||||||
|
wantPattern string
|
||||||
|
}{
|
||||||
|
{false, false, "position(message, {pattern:String}) > 0", "Timeout"},
|
||||||
|
{false, true, "positionCaseInsensitive(message, {pattern:String}) > 0", "Timeout"},
|
||||||
|
{true, false, "match(message, {pattern:String})", "Timeout"},
|
||||||
|
{true, true, "match(message, {pattern:String})", "(?i)Timeout"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
f := baseFilter()
|
||||||
|
f.Pattern = "Timeout"
|
||||||
|
f.Regex = c.regex
|
||||||
|
f.IgnoreCase = c.ignoreCase
|
||||||
|
q, err := Build(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(q.SQL, c.wantClause) {
|
||||||
|
t.Errorf("regex=%v i=%v: SQL missing %q:\n%s", c.regex, c.ignoreCase, c.wantClause, q.SQL)
|
||||||
|
}
|
||||||
|
if q.Params["pattern"] != c.wantPattern {
|
||||||
|
t.Errorf("regex=%v i=%v: pattern param = %q, want %q", c.regex, c.ignoreCase, q.Params["pattern"], c.wantPattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildFieldsDeterministic(t *testing.T) {
|
||||||
|
f := baseFilter()
|
||||||
|
f.Fields = map[string]string{"b": "2", "a": "1"}
|
||||||
|
q1, err := Build(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q2, _ := Build(f)
|
||||||
|
if q1.SQL != q2.SQL {
|
||||||
|
t.Error("field clause order not deterministic")
|
||||||
|
}
|
||||||
|
if !strings.Contains(q1.SQL, "fields[{fk0:String}] = {fv0:String}") ||
|
||||||
|
!strings.Contains(q1.SQL, "fields[{fk1:String}] = {fv1:String}") {
|
||||||
|
t.Errorf("field clauses missing:\n%s", q1.SQL)
|
||||||
|
}
|
||||||
|
if q1.Params["fk0"] != "a" || q1.Params["fv0"] != "1" || q1.Params["fk1"] != "b" || q1.Params["fv1"] != "2" {
|
||||||
|
t.Errorf("field params wrong: %v", q1.Params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelective(t *testing.T) {
|
||||||
|
f := baseFilter()
|
||||||
|
if f.Selective() {
|
||||||
|
t.Error("empty filter should not be selective")
|
||||||
|
}
|
||||||
|
for _, set := range []func(*Filter){
|
||||||
|
func(f *Filter) { f.Namespace = "x" },
|
||||||
|
func(f *Filter) { f.Host = "x" },
|
||||||
|
func(f *Filter) { f.App = "x" },
|
||||||
|
} {
|
||||||
|
g := baseFilter()
|
||||||
|
set(&g)
|
||||||
|
if !g.Selective() {
|
||||||
|
t.Errorf("filter %+v should be selective", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g := baseFilter()
|
||||||
|
g.Pod = "x"
|
||||||
|
g.Container = "x"
|
||||||
|
if g.Selective() {
|
||||||
|
t.Error("pod/container alone should not count as selective")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
TailInterval = 2 * time.Second
|
||||||
|
tailOverlap = 5 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// Tail streams the initial window then polls every interval. Each poll
|
||||||
|
// re-queries from a little before the last-seen timestamp and drops rows
|
||||||
|
// already emitted, so late-arriving rows inside the overlap still surface.
|
||||||
|
func Tail(ctx context.Context, c runner, f Filter, now func() time.Time, interval time.Duration, emit func(Row) error) error {
|
||||||
|
if now == nil {
|
||||||
|
now = time.Now
|
||||||
|
}
|
||||||
|
if interval <= 0 {
|
||||||
|
interval = TailInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[uint64]time.Time{}
|
||||||
|
lastTS := f.Since
|
||||||
|
track := func(r Row) error {
|
||||||
|
if ts := r.Time(); ts.After(lastTS) {
|
||||||
|
lastTS = ts
|
||||||
|
}
|
||||||
|
seen[r.Key()] = r.Time()
|
||||||
|
return emit(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := f
|
||||||
|
first.Until = now().UTC()
|
||||||
|
first.Limit = 0
|
||||||
|
if _, err := Page(ctx, c, first, 0, track); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
|
||||||
|
since := lastTS.Add(-tailOverlap)
|
||||||
|
if since.Before(f.Since) {
|
||||||
|
since = f.Since
|
||||||
|
}
|
||||||
|
pf := f
|
||||||
|
pf.Since = since
|
||||||
|
pf.Until = now().UTC()
|
||||||
|
pf.Limit = 0
|
||||||
|
if !pf.Since.Before(pf.Until) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, err := Page(ctx, c, pf, 0, func(r Row) error {
|
||||||
|
if _, dup := seen[r.Key()]; dup {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return track(r)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
floor := lastTS.Add(-2 * tailOverlap)
|
||||||
|
for k, ts := range seen {
|
||||||
|
if ts.Before(floor) {
|
||||||
|
delete(seen, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var durationRe = regexp.MustCompile(`^(\d+)([smhdw])$`)
|
||||||
|
|
||||||
|
var durationUnits = map[string]time.Duration{
|
||||||
|
"s": time.Second,
|
||||||
|
"m": time.Minute,
|
||||||
|
"h": time.Hour,
|
||||||
|
"d": 24 * time.Hour,
|
||||||
|
"w": 7 * 24 * time.Hour,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseTimeSpec accepts a relative duration (15m, 1h, 2d, 1w) meaning "that
|
||||||
|
// long before now", or an absolute RFC3339 timestamp.
|
||||||
|
func ParseTimeSpec(spec string, now time.Time) (time.Time, error) {
|
||||||
|
if spec == "" {
|
||||||
|
return time.Time{}, fmt.Errorf("empty time spec")
|
||||||
|
}
|
||||||
|
if m := durationRe.FindStringSubmatch(spec); m != nil {
|
||||||
|
n, err := strconv.ParseInt(m[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("invalid duration %q: %w", spec, err)
|
||||||
|
}
|
||||||
|
return now.Add(-time.Duration(n) * durationUnits[m[2]]), nil
|
||||||
|
}
|
||||||
|
t, err := time.Parse(time.RFC3339, spec)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("invalid time %q: use a duration (15m, 1h, 2d) or RFC3339", spec)
|
||||||
|
}
|
||||||
|
return t.UTC(), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
package chlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseTimeSpecDurations(t *testing.T) {
|
||||||
|
now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
||||||
|
cases := map[string]time.Time{
|
||||||
|
"15m": now.Add(-15 * time.Minute),
|
||||||
|
"1h": now.Add(-time.Hour),
|
||||||
|
"2d": now.Add(-48 * time.Hour),
|
||||||
|
"90s": now.Add(-90 * time.Second),
|
||||||
|
"1w": now.Add(-7 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
for spec, want := range cases {
|
||||||
|
got, err := ParseTimeSpec(spec, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: %v", spec, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !got.Equal(want) {
|
||||||
|
t.Errorf("%s: got %s, want %s", spec, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTimeSpecRFC3339(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
got, err := ParseTimeSpec("2026-08-23T10:30:00+10:00", now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := time.Date(2026, 8, 23, 0, 30, 0, 0, time.UTC)
|
||||||
|
if !got.Equal(want) {
|
||||||
|
t.Errorf("got %s, want %s", got, want)
|
||||||
|
}
|
||||||
|
if got.Location() != time.UTC {
|
||||||
|
t.Errorf("not normalized to UTC: %s", got.Location())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTimeSpecInvalid(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
for _, spec := range []string{"", "abc", "1x", "-5m", "2026-13-99", "1.5h"} {
|
||||||
|
if _, err := ParseTimeSpec(spec, now); err == nil {
|
||||||
|
t.Errorf("%q: expected error", spec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/clickhouse-tools/internal/chlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
var version = "dev"
|
||||||
|
|
||||||
|
const guardWindow = 6 * time.Hour
|
||||||
|
|
||||||
|
type commonFlags struct {
|
||||||
|
since string
|
||||||
|
until string
|
||||||
|
namespace string
|
||||||
|
host string
|
||||||
|
pod string
|
||||||
|
container string
|
||||||
|
app string
|
||||||
|
severity string
|
||||||
|
stream string
|
||||||
|
source string
|
||||||
|
limit uint64
|
||||||
|
format string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cf *commonFlags) register(cmd *cobra.Command, defaultLimit uint64) {
|
||||||
|
fl := cmd.Flags()
|
||||||
|
fl.StringVar(&cf.since, "since", "1h", "start of time range: duration ago (15m, 1h, 2d) or RFC3339")
|
||||||
|
fl.StringVar(&cf.until, "until", "", "end of time range: duration ago or RFC3339 (default now)")
|
||||||
|
fl.StringVarP(&cf.namespace, "namespace", "n", "", "filter: k8s namespace")
|
||||||
|
fl.StringVar(&cf.host, "host", "", "filter: host")
|
||||||
|
fl.StringVar(&cf.pod, "pod", "", "filter: pod name")
|
||||||
|
fl.StringVar(&cf.container, "container", "", "filter: container name")
|
||||||
|
fl.StringVar(&cf.app, "app", "", "filter: labels['app']")
|
||||||
|
fl.StringVar(&cf.severity, "severity", "", "filter: severity (case-insensitive)")
|
||||||
|
fl.StringVar(&cf.stream, "stream", "", "filter: stream (stdout/stderr)")
|
||||||
|
fl.StringVar(&cf.source, "source", "", "filter: source (k8s/vm)")
|
||||||
|
fl.Uint64Var(&cf.limit, "limit", defaultLimit, "maximum rows to print (0 = unlimited)")
|
||||||
|
fl.StringVar(&cf.format, "format", "text", "output format: text, json or logfmt")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cf *commonFlags) filter(now time.Time) (chlog.Filter, error) {
|
||||||
|
since, err := chlog.ParseTimeSpec(cf.since, now)
|
||||||
|
if err != nil {
|
||||||
|
return chlog.Filter{}, fmt.Errorf("--since: %w", err)
|
||||||
|
}
|
||||||
|
until := now
|
||||||
|
if cf.until != "" {
|
||||||
|
until, err = chlog.ParseTimeSpec(cf.until, now)
|
||||||
|
if err != nil {
|
||||||
|
return chlog.Filter{}, fmt.Errorf("--until: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return chlog.Filter{
|
||||||
|
Since: since.UTC(),
|
||||||
|
Until: until.UTC(),
|
||||||
|
Namespace: cf.namespace,
|
||||||
|
Host: cf.host,
|
||||||
|
Pod: cf.pod,
|
||||||
|
Container: cf.container,
|
||||||
|
App: cf.app,
|
||||||
|
Severity: cf.severity,
|
||||||
|
Stream: cf.stream,
|
||||||
|
Source: cf.source,
|
||||||
|
Limit: cf.limit,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func stdoutIsTTY() bool {
|
||||||
|
fi, err := os.Stdout.Stat()
|
||||||
|
return err == nil && fi.Mode()&os.ModeCharDevice != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cf *commonFlags) formatter() (chlog.Formatter, error) {
|
||||||
|
color := cf.format == "text" && stdoutIsTTY() && os.Getenv("NO_COLOR") == ""
|
||||||
|
return chlog.NewFormatter(cf.format, color)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCatCmd(use string) *cobra.Command {
|
||||||
|
cf := &commonFlags{}
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: use,
|
||||||
|
Short: "Print logs from the ClickHouse log store, oldest first",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
f, err := cf.filter(time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out, err := cf.formatter()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c := chlog.NewClient(chlog.ConfigFromEnv())
|
||||||
|
_, err = chlog.Page(cmd.Context(), c, f, 0, func(r chlog.Row) error {
|
||||||
|
return out(os.Stdout, r)
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cf.register(cmd, 10000)
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTailCmd(use string) *cobra.Command {
|
||||||
|
cf := &commonFlags{}
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: use,
|
||||||
|
Short: "Follow logs from the ClickHouse log store",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
cf.limit = 0
|
||||||
|
f, err := cf.filter(time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out, err := cf.formatter()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c := chlog.NewClient(chlog.ConfigFromEnv())
|
||||||
|
err = chlog.Tail(cmd.Context(), c, f, nil, chlog.TailInterval, func(r chlog.Row) error {
|
||||||
|
return out(os.Stdout, r)
|
||||||
|
})
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cf.register(cmd, 0)
|
||||||
|
cmd.Flags().MarkHidden("until")
|
||||||
|
cmd.Flags().MarkHidden("limit")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGrepCmd(use string) *cobra.Command {
|
||||||
|
cf := &commonFlags{}
|
||||||
|
var (
|
||||||
|
regex bool
|
||||||
|
ignoreCase bool
|
||||||
|
fields []string
|
||||||
|
force bool
|
||||||
|
)
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: use + " <pattern>",
|
||||||
|
Short: "Search log messages in the ClickHouse log store",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
f, err := cf.filter(time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.Pattern = args[0]
|
||||||
|
f.Regex = regex
|
||||||
|
f.IgnoreCase = ignoreCase
|
||||||
|
f.Fields, err = parseFields(fields)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !f.Selective() && f.Until.Sub(f.Since) > guardWindow && !force {
|
||||||
|
return fmt.Errorf("unfiltered search over %s scans the whole table (no message index, ~281M rows/day); add --namespace/--host/--app, shrink --since to 6h or less, or pass --force",
|
||||||
|
f.Until.Sub(f.Since).Round(time.Minute))
|
||||||
|
}
|
||||||
|
out, err := cf.formatter()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c := chlog.NewClient(chlog.ConfigFromEnv())
|
||||||
|
_, err = chlog.Page(cmd.Context(), c, f, 0, func(r chlog.Row) error {
|
||||||
|
return out(os.Stdout, r)
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cf.register(cmd, 10000)
|
||||||
|
fl := cmd.Flags()
|
||||||
|
fl.BoolVar(®ex, "regex", false, "treat pattern as an RE2 regular expression")
|
||||||
|
fl.BoolVarP(&ignoreCase, "ignore-case", "i", false, "case-insensitive match")
|
||||||
|
fl.StringArrayVar(&fields, "fields", nil, "filter on structured fields: key=value (repeatable)")
|
||||||
|
fl.BoolVar(&force, "force", false, "allow an unfiltered search wider than 6h")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseFields(kvs []string) (map[string]string, error) {
|
||||||
|
if len(kvs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
m := make(map[string]string, len(kvs))
|
||||||
|
for _, kv := range kvs {
|
||||||
|
k, v, ok := strings.Cut(kv, "=")
|
||||||
|
if !ok || k == "" {
|
||||||
|
return nil, fmt.Errorf("--fields %q: want key=value", kv)
|
||||||
|
}
|
||||||
|
m[k] = v
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRootCmd() *cobra.Command {
|
||||||
|
root := &cobra.Command{
|
||||||
|
Use: "chlog",
|
||||||
|
Short: "CLI for the ClickHouse log store (logs.raw)",
|
||||||
|
Version: version,
|
||||||
|
SilenceUsage: true,
|
||||||
|
SilenceErrors: true,
|
||||||
|
}
|
||||||
|
root.AddCommand(newCatCmd("cat"), newTailCmd("tail"), newGrepCmd("grep"))
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func entrypoint() *cobra.Command {
|
||||||
|
var cmd *cobra.Command
|
||||||
|
switch filepath.Base(os.Args[0]) {
|
||||||
|
case "chcat":
|
||||||
|
cmd = newCatCmd("chcat")
|
||||||
|
case "chtail":
|
||||||
|
cmd = newTailCmd("chtail")
|
||||||
|
case "chgrep":
|
||||||
|
cmd = newGrepCmd("chgrep")
|
||||||
|
default:
|
||||||
|
return newRootCmd()
|
||||||
|
}
|
||||||
|
cmd.Version = version
|
||||||
|
cmd.SilenceUsage = true
|
||||||
|
cmd.SilenceErrors = true
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
if err := entrypoint().ExecuteContext(ctx); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "error:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseFields(t *testing.T) {
|
||||||
|
m, err := parseFields([]string{"a=1", "b=x=y"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m["a"] != "1" || m["b"] != "x=y" {
|
||||||
|
t.Errorf("m = %v", m)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"noequals", "=v"} {
|
||||||
|
if _, err := parseFields([]string{bad}); err == nil {
|
||||||
|
t.Errorf("%q: expected error", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m, _ := parseFields(nil); m != nil {
|
||||||
|
t.Error("nil input should give nil map")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommonFlagsFilterDefaults(t *testing.T) {
|
||||||
|
cf := &commonFlags{since: "1h", format: "text"}
|
||||||
|
now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
||||||
|
f, err := cf.filter(now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !f.Since.Equal(now.Add(-time.Hour)) {
|
||||||
|
t.Errorf("since = %s", f.Since)
|
||||||
|
}
|
||||||
|
if !f.Until.Equal(now) {
|
||||||
|
t.Errorf("until = %s", f.Until)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGrepGuardBlocksWideUnfilteredSearch(t *testing.T) {
|
||||||
|
cmd := newGrepCmd("chgrep")
|
||||||
|
cmd.SetArgs([]string{"--since", "24h", "needle"})
|
||||||
|
var out strings.Builder
|
||||||
|
cmd.SetOut(&out)
|
||||||
|
cmd.SetErr(&out)
|
||||||
|
err := cmd.Execute()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "--force") {
|
||||||
|
t.Fatalf("expected guard error mentioning --force, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGrepGuardAllowsFilteredSearch(t *testing.T) {
|
||||||
|
// A namespace filter disables the guard; the query then fails on the
|
||||||
|
// unreachable server rather than the guard, proving the guard passed.
|
||||||
|
t.Setenv("CH_URL", "http://127.0.0.1:1")
|
||||||
|
cmd := newGrepCmd("chgrep")
|
||||||
|
cmd.SetArgs([]string{"--since", "24h", "--namespace", "logging", "needle"})
|
||||||
|
var out strings.Builder
|
||||||
|
cmd.SetOut(&out)
|
||||||
|
cmd.SetErr(&out)
|
||||||
|
err := cmd.Execute()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected connection error")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "--force") {
|
||||||
|
t.Fatalf("guard should not trigger with a namespace filter: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGrepGuardAllowsShortWindow(t *testing.T) {
|
||||||
|
t.Setenv("CH_URL", "http://127.0.0.1:1")
|
||||||
|
cmd := newGrepCmd("chgrep")
|
||||||
|
cmd.SetArgs([]string{"--since", "1h", "needle"})
|
||||||
|
err := cmd.Execute()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected connection error")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "--force") {
|
||||||
|
t.Fatalf("guard should not trigger for 1h window: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEntrypointDispatch(t *testing.T) {
|
||||||
|
root := newRootCmd()
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, c := range root.Commands() {
|
||||||
|
names[c.Name()] = true
|
||||||
|
}
|
||||||
|
for _, want := range []string{"cat", "tail", "grep"} {
|
||||||
|
if !names[want] {
|
||||||
|
t.Errorf("chlog missing subcommand %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# nfpm config for the clickhouse-tools RPM.
|
||||||
|
# Rendered through envsubst (see scripts/build-rpm.sh) then fed to `nfpm pkg`.
|
||||||
|
|
||||||
|
name: ${PACKAGE_NAME}
|
||||||
|
version: ${PACKAGE_VERSION}
|
||||||
|
release: ${PACKAGE_RELEASE}
|
||||||
|
arch: ${PACKAGE_ARCH}
|
||||||
|
platform: ${PACKAGE_PLATFORM}
|
||||||
|
section: default
|
||||||
|
priority: extra
|
||||||
|
description: "${PACKAGE_DESCRIPTION}"
|
||||||
|
|
||||||
|
maintainer: ${PACKAGE_MAINTAINER}
|
||||||
|
homepage: ${PACKAGE_HOMEPAGE}
|
||||||
|
license: ${PACKAGE_LICENSE}
|
||||||
|
|
||||||
|
disable_globbing: false
|
||||||
|
|
||||||
|
contents:
|
||||||
|
- src: dist/chlog
|
||||||
|
dst: /usr/bin/chlog
|
||||||
|
file_info:
|
||||||
|
mode: 0755
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
|
||||||
|
# chcat/chtail/chgrep dispatch on argv[0] inside the chlog binary.
|
||||||
|
- src: /usr/bin/chlog
|
||||||
|
dst: /usr/bin/chcat
|
||||||
|
type: symlink
|
||||||
|
- src: /usr/bin/chlog
|
||||||
|
dst: /usr/bin/chtail
|
||||||
|
type: symlink
|
||||||
|
- src: /usr/bin/chlog
|
||||||
|
dst: /usr/bin/chgrep
|
||||||
|
type: symlink
|
||||||
|
|
||||||
|
# Shell completions (generated by scripts/build-rpm.sh before packaging).
|
||||||
|
- src: dist/completions/chlog.bash
|
||||||
|
dst: /usr/share/bash-completion/completions/chlog
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/_chlog
|
||||||
|
dst: /usr/share/zsh/site-functions/_chlog
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chlog.fish
|
||||||
|
dst: /usr/share/fish/vendor_completions.d/chlog.fish
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chcat.bash
|
||||||
|
dst: /usr/share/bash-completion/completions/chcat
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/_chcat
|
||||||
|
dst: /usr/share/zsh/site-functions/_chcat
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chcat.fish
|
||||||
|
dst: /usr/share/fish/vendor_completions.d/chcat.fish
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chtail.bash
|
||||||
|
dst: /usr/share/bash-completion/completions/chtail
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/_chtail
|
||||||
|
dst: /usr/share/zsh/site-functions/_chtail
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chtail.fish
|
||||||
|
dst: /usr/share/fish/vendor_completions.d/chtail.fish
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chgrep.bash
|
||||||
|
dst: /usr/share/bash-completion/completions/chgrep
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/_chgrep
|
||||||
|
dst: /usr/share/zsh/site-functions/_chgrep
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
|
- src: dist/completions/chgrep.fish
|
||||||
|
dst: /usr/share/fish/vendor_completions.d/chgrep.fish
|
||||||
|
file_info:
|
||||||
|
mode: 0644
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Package the (already built) chlog binary into an RPM with nfpm, bundling
|
||||||
|
# chcat/chtail/chgrep symlinks and generated bash/zsh/fish shell completions.
|
||||||
|
# Usage: scripts/build-rpm.sh [version] (version defaults to $CI_COMMIT_TAG)
|
||||||
|
#
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "${ROOT_DIR}"
|
||||||
|
|
||||||
|
VERSION="${1:-${CI_COMMIT_TAG:-0.0.0-dev}}"
|
||||||
|
VERSION="${VERSION#v}" # strip a leading v
|
||||||
|
BINARY="chlog"
|
||||||
|
NAMES=(chlog chcat chtail chgrep)
|
||||||
|
DIST="dist"
|
||||||
|
|
||||||
|
if [ ! -f "${DIST}/${BINARY}" ]; then
|
||||||
|
echo "ERROR: ${DIST}/${BINARY} not found; run 'make build' first" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for l in chcat chtail chgrep; do
|
||||||
|
ln -sf "${BINARY}" "${DIST}/${l}"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Generate shell completions per entrypoint name so they always match the
|
||||||
|
# shipped flags/subcommands (each name dispatches to its own command tree).
|
||||||
|
COMP_DIR="${DIST}/completions"
|
||||||
|
mkdir -p "${COMP_DIR}"
|
||||||
|
for b in "${NAMES[@]}"; do
|
||||||
|
"./${DIST}/${b}" completion bash >"${COMP_DIR}/${b}.bash"
|
||||||
|
"./${DIST}/${b}" completion zsh >"${COMP_DIR}/_${b}"
|
||||||
|
"./${DIST}/${b}" completion fish >"${COMP_DIR}/${b}.fish"
|
||||||
|
done
|
||||||
|
|
||||||
|
export PACKAGE_NAME="clickhouse-tools"
|
||||||
|
export PACKAGE_VERSION="${VERSION}"
|
||||||
|
export PACKAGE_RELEASE="1"
|
||||||
|
export PACKAGE_ARCH="amd64"
|
||||||
|
export PACKAGE_PLATFORM="linux"
|
||||||
|
export PACKAGE_DESCRIPTION="CLI tools for the ClickHouse log store: chlog with chcat (print), chtail (follow) and chgrep (search) entrypoints"
|
||||||
|
export PACKAGE_MAINTAINER="Ben Vincent <ben@unkin.net>"
|
||||||
|
export PACKAGE_HOMEPAGE="https://git.unkin.net/unkin/clickhouse-tools"
|
||||||
|
export PACKAGE_LICENSE="MIT"
|
||||||
|
|
||||||
|
envsubst <packaging/nfpm.yaml >"${DIST}/nfpm.yaml"
|
||||||
|
nfpm pkg --config "${DIST}/nfpm.yaml" --target "${DIST}" --packager rpm
|
||||||
|
|
||||||
|
echo "Built:"
|
||||||
|
ls -1 "${DIST}"/*.rpm
|
||||||
Reference in New Issue
Block a user