clickhouse-tools/internal/chlog/query.go

package chlog

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
)

const Table = "logs.raw"

// Filter describes one bounded query against logs.raw. Since/Until are
// mandatory: the table has no text index, so every query must be time-bounded.
type Filter struct {
	Since time.Time
	Until time.Time

	Namespace string
	Host      string
	Pod       string
	Container string
	App       string
	Severity  string
	Stream    string
	Source    string

	Pattern    string
	Regex      bool
	IgnoreCase bool
	Fields     map[string]string

	Limit uint64
}

// Selective reports whether the filter narrows the scan enough to be cheap:
// any of namespace, host, or app restricts to a small slice of the table.
func (f Filter) Selective() bool {
	return f.Namespace != "" || f.Host != "" || f.App != ""
}

type Query struct {
	SQL    string
	Params map[string]string
}

const selectColumns = "timestamp, host, source, namespace, pod, container, stream, severity, message, labels, fields"

func chTime(t time.Time) string {
	return strconv.FormatInt(t.UnixMilli(), 10)
}

// Build renders a fully parameterized query. All user-supplied values travel
// as HTTP {name:Type} parameters, never interpolated into the SQL text.
func Build(f Filter) (Query, error) {
	if f.Since.IsZero() || f.Until.IsZero() {
		return Query{}, fmt.Errorf("query must be time-bounded: since/until missing")
	}
	if !f.Since.Before(f.Until) {
		return Query{}, fmt.Errorf("empty time range: since %s is not before until %s",
			f.Since.UTC().Format(time.RFC3339), f.Until.UTC().Format(time.RFC3339))
	}

	params := map[string]string{
		"since_ms": chTime(f.Since),
		"until_ms": chTime(f.Until),
	}
	where := []string{
		"timestamp >= fromUnixTimestamp64Milli({since_ms:Int64})",
		"timestamp < fromUnixTimestamp64Milli({until_ms:Int64})",
	}

	addEq := func(column, name, value string) {
		if value == "" {
			return
		}
		where = append(where, fmt.Sprintf("%s = {%s:String}", column, name))
		params[name] = value
	}
	addEq("namespace", "ns", f.Namespace)
	addEq("host", "host", f.Host)
	addEq("pod", "pod", f.Pod)
	addEq("container", "container", f.Container)
	addEq("stream", "stream", f.Stream)
	addEq("source", "source", f.Source)
	addEq("labels['app']", "app", f.App)

	if f.Severity != "" {
		where = append(where, "lowerUTF8(severity) = {severity:String}")
		params["severity"] = strings.ToLower(f.Severity)
	}

	if f.Pattern != "" {
		switch {
		case f.Regex:
			pat := f.Pattern
			if f.IgnoreCase {
				pat = "(?i)" + pat
			}
			where = append(where, "match(message, {pattern:String})")
			params["pattern"] = pat
		case f.IgnoreCase:
			where = append(where, "positionCaseInsensitive(message, {pattern:String}) > 0")
			params["pattern"] = f.Pattern
		default:
			where = append(where, "position(message, {pattern:String}) > 0")
			params["pattern"] = f.Pattern
		}
	}

	keys := make([]string, 0, len(f.Fields))
	for k := range f.Fields {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for i, k := range keys {
		kn := fmt.Sprintf("fk%d", i)
		vn := fmt.Sprintf("fv%d", i)
		where = append(where, fmt.Sprintf("fields[{%s:String}] = {%s:String}", kn, vn))
		params[kn] = k
		params[vn] = f.Fields[k]
	}

	sql := fmt.Sprintf("SELECT %s FROM %s WHERE %s ORDER BY timestamp ASC",
		selectColumns, Table, strings.Join(where, " AND "))
	if f.Limit > 0 {
		sql += " LIMIT {limit:UInt64}"
		params["limit"] = strconv.FormatUint(f.Limit, 10)
	}
	return Query{SQL: sql, Params: params}, nil
}