unkin-agent
415bf0cce1
Single Go binary for the ClickHouse log store (logs.raw): chlog with cat/tail/grep subcommands, plus chcat/chtail/chgrep argv[0]-dispatched symlink entrypoints. Every query is time-bounded and fully parameterized; chgrep guards wide unfiltered scans. Ships nfpm RPM with completions and woodpecker PR/tag pipelines mirroring node-lookup.
280 lines
7.3 KiB
Go
280 lines
7.3 KiB
Go
package chlog
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func chTS(t time.Time) string {
|
|
return t.UTC().Format("2006-01-02 15:04:05.000")
|
|
}
|
|
|
|
func mkRow(ts time.Time, msg string) Row {
|
|
return Row{
|
|
Timestamp: chTS(ts),
|
|
Host: "node1",
|
|
Source: "k8s",
|
|
Namespace: "logging",
|
|
Pod: "vector-abc",
|
|
Container: "vector",
|
|
Stream: "stdout",
|
|
Message: msg,
|
|
}
|
|
}
|
|
|
|
// fakeCH serves the ClickHouse HTTP contract used by Client: JSONEachRow rows
|
|
// from an in-memory store, honoring the since/until/limit query parameters.
|
|
type fakeCH struct {
|
|
mu sync.Mutex
|
|
rows []Row
|
|
queries int
|
|
lastUser string
|
|
lastPass string
|
|
lastSQL string
|
|
}
|
|
|
|
func (f *fakeCH) add(rows ...Row) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.rows = append(f.rows, rows...)
|
|
}
|
|
|
|
func (f *fakeCH) handler(t *testing.T) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.queries++
|
|
f.lastUser = r.Header.Get("X-ClickHouse-User")
|
|
f.lastPass = r.Header.Get("X-ClickHouse-Key")
|
|
body := make([]byte, r.ContentLength)
|
|
r.Body.Read(body)
|
|
f.lastSQL = string(body)
|
|
|
|
q := r.URL.Query()
|
|
since, err1 := strconv.ParseInt(q.Get("param_since_ms"), 10, 64)
|
|
until, err2 := strconv.ParseInt(q.Get("param_until_ms"), 10, 64)
|
|
if err1 != nil || err2 != nil {
|
|
t.Errorf("query missing time bound params: %s", r.URL.RawQuery)
|
|
http.Error(w, "unbounded query", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var limit int64 = -1
|
|
if s := q.Get("param_limit"); s != "" {
|
|
limit, _ = strconv.ParseInt(s, 10, 64)
|
|
}
|
|
var sent int64
|
|
enc := json.NewEncoder(w)
|
|
for _, row := range f.rows {
|
|
ms := row.Time().UnixMilli()
|
|
if ms < since || ms >= until {
|
|
continue
|
|
}
|
|
if limit >= 0 && sent >= limit {
|
|
break
|
|
}
|
|
enc.Encode(row)
|
|
sent++
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestClientRun(t *testing.T) {
|
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
|
fake := &fakeCH{}
|
|
fake.add(mkRow(base, "hello"), mkRow(base.Add(time.Second), "world"))
|
|
srv := httptest.NewServer(fake.handler(t))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader", Password: "secret"})
|
|
q, err := Build(Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Minute)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got []Row
|
|
if err := c.Run(context.Background(), q, func(r Row) error {
|
|
got = append(got, r)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 2 || got[0].Message != "hello" || got[1].Message != "world" {
|
|
t.Fatalf("rows = %+v", got)
|
|
}
|
|
if fake.lastUser != "logreader" || fake.lastPass != "secret" {
|
|
t.Errorf("auth headers: user=%q pass set=%v", fake.lastUser, fake.lastPass != "")
|
|
}
|
|
if !strings.Contains(fake.lastSQL, "FROM logs.raw") {
|
|
t.Errorf("SQL body not sent: %q", fake.lastSQL)
|
|
}
|
|
if got[0].Time().IsZero() {
|
|
t.Error("timestamp did not parse")
|
|
}
|
|
}
|
|
|
|
func TestClientRunHTTPError(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Code: 159. DB::Exception: Timeout exceeded", http.StatusInternalServerError)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
|
q, _ := Build(Filter{Since: time.Now().Add(-time.Hour), Until: time.Now()})
|
|
err := c.Run(context.Background(), q, func(Row) error { return nil })
|
|
if err == nil || !strings.Contains(err.Error(), "HTTP 500") || !strings.Contains(err.Error(), "Timeout exceeded") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPagePagesAndDedupes(t *testing.T) {
|
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
|
fake := &fakeCH{}
|
|
// 25 rows, including three rows sharing one boundary timestamp.
|
|
for i := 0; i < 25; i++ {
|
|
ts := base.Add(time.Duration(i) * time.Second)
|
|
if i >= 10 && i < 13 {
|
|
ts = base.Add(10 * time.Second)
|
|
}
|
|
fake.add(mkRow(ts, fmt.Sprintf("msg-%d", i)))
|
|
}
|
|
srv := httptest.NewServer(fake.handler(t))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour)}
|
|
var got []string
|
|
n, err := Page(context.Background(), c, f, 7, func(r Row) error {
|
|
got = append(got, r.Message)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 25 || len(got) != 25 {
|
|
t.Fatalf("emitted %d rows (%d reported), want 25: %v", len(got), n, got)
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, m := range got {
|
|
if seen[m] {
|
|
t.Fatalf("duplicate row %q emitted", m)
|
|
}
|
|
seen[m] = true
|
|
}
|
|
if fake.queries < 4 {
|
|
t.Errorf("expected multiple paged queries, got %d", fake.queries)
|
|
}
|
|
}
|
|
|
|
func TestPageHonorsLimit(t *testing.T) {
|
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
|
fake := &fakeCH{}
|
|
for i := 0; i < 30; i++ {
|
|
fake.add(mkRow(base.Add(time.Duration(i)*time.Second), fmt.Sprintf("msg-%d", i)))
|
|
}
|
|
srv := httptest.NewServer(fake.handler(t))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour), Limit: 12}
|
|
var got []string
|
|
n, err := Page(context.Background(), c, f, 5, func(r Row) error {
|
|
got = append(got, r.Message)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 12 || len(got) != 12 || got[0] != "msg-0" || got[11] != "msg-11" {
|
|
t.Fatalf("emitted %d rows: %v", len(got), got)
|
|
}
|
|
}
|
|
|
|
func TestPageSingleMillisecondBurst(t *testing.T) {
|
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
|
fake := &fakeCH{}
|
|
for i := 0; i < 6; i++ {
|
|
fake.add(mkRow(base, fmt.Sprintf("burst-%d", i)))
|
|
}
|
|
fake.add(mkRow(base.Add(time.Second), "after"))
|
|
srv := httptest.NewServer(fake.handler(t))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
|
f := Filter{Since: base.Add(-time.Minute), Until: base.Add(time.Hour)}
|
|
var got []string
|
|
_, err := Page(context.Background(), c, f, 3, func(r Row) error {
|
|
got = append(got, r.Message)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 7 || got[6] != "after" {
|
|
t.Fatalf("rows = %v", got)
|
|
}
|
|
}
|
|
|
|
func TestTailDedupesAcrossPolls(t *testing.T) {
|
|
base := time.Date(2026, 8, 23, 5, 0, 0, 0, time.UTC)
|
|
fake := &fakeCH{}
|
|
fake.add(mkRow(base, "initial-1"), mkRow(base.Add(time.Second), "initial-2"))
|
|
srv := httptest.NewServer(fake.handler(t))
|
|
defer srv.Close()
|
|
|
|
c := NewClient(Config{URL: srv.URL, User: "logreader"})
|
|
|
|
var mu sync.Mutex
|
|
var got []string
|
|
polls := 0
|
|
now := func() time.Time {
|
|
return base.Add(time.Duration(10+polls) * time.Second)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
emit := func(r Row) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
got = append(got, r.Message)
|
|
return nil
|
|
}
|
|
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- Tail(ctx, c, Filter{Since: base.Add(-time.Minute)}, func() time.Time {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
polls++
|
|
return now()
|
|
}, 10*time.Millisecond, emit)
|
|
}()
|
|
|
|
// Late-arriving row inside the overlap window plus a genuinely new row.
|
|
time.Sleep(50 * time.Millisecond)
|
|
fake.add(mkRow(base.Add(500*time.Millisecond), "late"), mkRow(base.Add(2*time.Second), "new"))
|
|
time.Sleep(100 * time.Millisecond)
|
|
cancel()
|
|
if err := <-done; err != context.Canceled {
|
|
t.Fatalf("tail err = %v", err)
|
|
}
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
counts := map[string]int{}
|
|
for _, m := range got {
|
|
counts[m]++
|
|
}
|
|
for _, want := range []string{"initial-1", "initial-2", "late", "new"} {
|
|
if counts[want] != 1 {
|
|
t.Errorf("row %q emitted %d times; all: %v", want, counts[want], got)
|
|
}
|
|
}
|
|
}
|