unkinben
da4a66046a
Why: extend the Tier-1 survey with 7 more high-value log sources so they parse into logs.raw columns/.fields for real querying instead of the generic catch-all. **Stacks on #318 — merge after it.** How: - 7 mutually-exclusive app_route conditions + parse transforms into the ClickHouse sink: **bind_query** (k8s bind-* + VM named), **rancher_audit** (cattle-system sidecar JSON), **cnpg_pg** (ONE transform for all 10 CNPG clusters via the `.postgres` container), **gitea** (router+access, k8s+VM), **puppet** (openvoxserver/openvoxdb logback + access), **litellm** (JSON request logs), **postfix** (per-line maillog). - Carve `.postgres` out of the Tier-1 authentik route + new puppet/gitea/litellm routes so the single cnpg_pg route claims every CNPG pod without double-insert (keeps app_route mutually exclusive). Catch-all intact. - Companion k8s flips in this PR: litellm `JSON_LOGS=True`; bind `querylog yes` on both bind-internal BindClusters; gitea router+access logging to stdout. Rancher auditLog was already on. - 15 new `vector test` cases (routing + field extraction + authentik-postgres→cnpg exclusivity proof); all 35 green (vector 0.57). Fields go into the existing `fields Map(String,String)` — no DDL change. Puppet-side follow-ups (out of scope for argocd): enable named query logging (profiles/dns/server.pp); ship the VM vector rollout with `.file`/`.SYSLOG_IDENTIFIER` tags for named/gitea/puppetserver(+multiline logback join)/postfix maillog. https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --------- Co-authored-by: Ben Vincent <neotheo@gmail.com> Reviewed-on: #320 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
669 lines
24 KiB
YAML
669 lines
24 KiB
YAML
---
|
|
# `vector test` unit tests for the aggregator transforms. Merged with
|
|
# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the
|
|
# per-app parsing follow-ups extend: add a test per new transform here.
|
|
tests:
|
|
- name: subject_routes_k8s_vs_vm
|
|
inputs:
|
|
- insert_at: route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.shop.web"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: route.k8s
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.message, "routed")
|
|
|
|
- name: k8s_log_is_normalised
|
|
inputs:
|
|
- insert_at: k8s_shape
|
|
type: log
|
|
log_fields:
|
|
message: "hello from pod"
|
|
stream: "stdout"
|
|
timestamp: "2026-07-27T00:00:00Z"
|
|
kubernetes.pod_name: "web-abc"
|
|
kubernetes.pod_namespace: "shop"
|
|
kubernetes.container_name: "web"
|
|
kubernetes.pod_node_name: "node-1"
|
|
outputs:
|
|
- extract_from: k8s_shape
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "shop")
|
|
assert_eq!(.pod, "web-abc")
|
|
assert_eq!(.container, "web")
|
|
assert_eq!(.host, "node-1")
|
|
assert_eq!(.stream, "stdout")
|
|
assert_eq!(.message, "hello from pod")
|
|
|
|
- name: vm_log_is_normalised
|
|
inputs:
|
|
- insert_at: vm_shape
|
|
type: log
|
|
log_fields:
|
|
message: "sshd started"
|
|
host: "vm-db-1"
|
|
severity: "info"
|
|
role: "database"
|
|
outputs:
|
|
- extract_from: vm_shape
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.host, "vm-db-1")
|
|
assert_eq!(.severity, "info")
|
|
assert_eq!(.message, "sshd started")
|
|
assert_eq!(.labels.role, "database")
|
|
|
|
# --- catch-all preservation: an unclaimed k8s event still flows app_route ->
|
|
# generic route -> k8s_shape (proves the two-stage chain keeps the fallback) ---
|
|
- name: unclaimed_k8s_falls_through_to_generic
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.shop.web"
|
|
message: "plain app log"
|
|
outputs:
|
|
- extract_from: route.k8s
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.message, "plain app log")
|
|
|
|
# --- Tier-1: Authentik SSO (k8s, LIVE NOW) ---
|
|
- name: authentik_routes_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.authentik.server"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.authentik
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: authentik_parse_extracts_event
|
|
inputs:
|
|
- insert_at: authentik_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.authentik.server"
|
|
stream: "stdout"
|
|
kubernetes.pod_namespace: "authentik"
|
|
kubernetes.container_name: "server"
|
|
kubernetes.pod_node_name: "node-2"
|
|
message: '{"event":"login","action":"login","user":"alice","client_ip":"203.0.113.9","result":"success","level":"info","logger":"authentik.events","timestamp":"2026-07-27T00:00:00Z"}'
|
|
outputs:
|
|
- extract_from: authentik_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "authentik")
|
|
assert_eq!(.container, "server")
|
|
assert_eq!(.severity, "info")
|
|
assert_eq!(.message, "login")
|
|
assert_eq!(.labels.app, "authentik")
|
|
assert_eq!(.fields.event, "login")
|
|
assert_eq!(.fields.action, "login")
|
|
assert_eq!(.fields.user, "alice")
|
|
assert_eq!(.fields.client_ip, "203.0.113.9")
|
|
assert_eq!(.fields.result, "success")
|
|
|
|
# --- Tier-1: Traefik ingress (k8s, JSON access logs) ---
|
|
- name: traefik_routes_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.traefik-system.traefik"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.traefik
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: traefik_parse_extracts_access_fields
|
|
inputs:
|
|
- insert_at: traefik_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.traefik-system.traefik"
|
|
kubernetes.pod_namespace: "traefik-system"
|
|
kubernetes.container_name: "traefik"
|
|
kubernetes.pod_node_name: "node-3"
|
|
message: '{"RouterName":"web@kubernetes","ServiceName":"shop-svc@kubernetes","RequestMethod":"GET","RequestPath":"/api","RequestHost":"shop.example.net","RequestProtocol":"HTTP/1.1","DownstreamStatus":200,"Duration":5000000,"ClientHost":"203.0.113.5","StartUTC":"2026-07-27T00:00:00Z"}'
|
|
outputs:
|
|
- extract_from: traefik_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "traefik-system")
|
|
assert_eq!(.labels.app, "traefik")
|
|
assert_eq!(.message, "GET /api 200")
|
|
assert_eq!(.fields.route, "web@kubernetes")
|
|
assert_eq!(.fields.service, "shop-svc@kubernetes")
|
|
assert_eq!(.fields.method, "GET")
|
|
assert_eq!(.fields.path, "/api")
|
|
assert_eq!(.fields.host, "shop.example.net")
|
|
assert_eq!(.fields.status, "200")
|
|
assert_eq!(.fields.duration_ms, "5")
|
|
assert_eq!(.fields.client_ip, "203.0.113.5")
|
|
|
|
# --- Tier-1: Vault/OpenBao file audit (VM, awaiting VM vector) ---
|
|
- name: vault_routes_by_file
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.vault1_syd1"
|
|
file: "/var/log/vault_audit.log"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.vault
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: vault_parse_extracts_request
|
|
inputs:
|
|
- insert_at: vault_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.vault1_syd1"
|
|
host: "vault1"
|
|
file: "/var/log/vault_audit.log"
|
|
message: '{"time":"2026-07-27T00:00:00Z","type":"response","auth":{"display_name":"token"},"request":{"operation":"read","path":"secret/data/app","remote_address":"10.0.0.9"},"error":""}'
|
|
outputs:
|
|
- extract_from: vault_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.host, "vault1")
|
|
assert_eq!(.labels.app, "vault")
|
|
assert_eq!(.message, "read secret/data/app")
|
|
assert_eq!(.fields.type, "response")
|
|
assert_eq!(.fields.display_name, "token")
|
|
assert_eq!(.fields.operation, "read")
|
|
assert_eq!(.fields.path, "secret/data/app")
|
|
assert_eq!(.fields.remote_address, "10.0.0.9")
|
|
|
|
# --- Tier-1: nginx access (VM, awaiting VM vector) ---
|
|
- name: nginx_access_routes_by_file
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.web1_syd1"
|
|
file: "/var/log/nginx/shop_access.log"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.nginx_access
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: nginx_access_parse_extracts_combined
|
|
inputs:
|
|
- insert_at: nginx_access_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.web1_syd1"
|
|
host: "web1"
|
|
file: "/var/log/nginx/shop_access.log"
|
|
message: '192.0.2.10 - - [27/Jul/2026:00:00:00 +0000] "GET /index.html HTTP/1.1" 200 1024 "https://ref.example/" "Mozilla/5.0" 0.012'
|
|
outputs:
|
|
- extract_from: nginx_access_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.stream, "access")
|
|
assert_eq!(.labels.log_type, "access")
|
|
assert_eq!(.fields.client_ip, "192.0.2.10")
|
|
assert_eq!(.fields.method, "GET")
|
|
assert_eq!(.fields.path, "/index.html")
|
|
assert_eq!(.fields.status, "200")
|
|
assert_eq!(.fields.bytes, "1024")
|
|
assert_eq!(.fields.referer, "https://ref.example/")
|
|
assert_eq!(.fields.user_agent, "Mozilla/5.0")
|
|
assert_eq!(.fields.request_time, "0.012")
|
|
|
|
# --- Tier-1: nginx error (VM, awaiting VM vector) ---
|
|
- name: nginx_error_routes_by_file
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.web1_syd1"
|
|
file: "/var/log/nginx/shop_error.log"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.nginx_error
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: nginx_error_parse_extracts_fields
|
|
inputs:
|
|
- insert_at: nginx_error_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.web1_syd1"
|
|
host: "web1"
|
|
file: "/var/log/nginx/shop_error.log"
|
|
message: '2026/07/27 00:00:00 [error] 1234#0: *5 open() "/var/www/x" failed (2: No such file or directory), client: 192.0.2.20, server: shop, request: "GET / HTTP/1.1", host: "shop"'
|
|
outputs:
|
|
- extract_from: nginx_error_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.stream, "error")
|
|
assert_eq!(.severity, "error")
|
|
assert_eq!(.labels.log_type, "error")
|
|
assert_eq!(.fields.level, "error")
|
|
assert_eq!(.fields.pid, "1234")
|
|
assert_eq!(.fields.cid, "5")
|
|
assert_eq!(.fields.client_ip, "192.0.2.20")
|
|
|
|
# --- Tier-1: HAProxy httplog (VM journald, awaiting VM vector) ---
|
|
- name: haproxy_routes_by_identifier
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.halb1_syd1"
|
|
SYSLOG_IDENTIFIER: "haproxy"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.haproxy
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: haproxy_parse_extracts_timers
|
|
inputs:
|
|
- insert_at: haproxy_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.halb1_syd1"
|
|
host: "halb1"
|
|
SYSLOG_IDENTIFIER: "haproxy"
|
|
message: '192.0.2.30:54321 [27/Jul/2026:00:00:00.123] fe_http be_app/app1 10/0/1/2/13 200 512 - - ---- 5/4/3/2/0 0/0 "GET /health HTTP/1.1"'
|
|
outputs:
|
|
- extract_from: haproxy_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.labels.app, "haproxy")
|
|
assert_eq!(.fields.client_ip, "192.0.2.30")
|
|
assert_eq!(.fields.frontend, "fe_http")
|
|
assert_eq!(.fields.backend, "be_app")
|
|
assert_eq!(.fields.server, "app1")
|
|
assert_eq!(.fields.tq, "10")
|
|
assert_eq!(.fields.tw, "0")
|
|
assert_eq!(.fields.tc, "1")
|
|
assert_eq!(.fields.tr, "2")
|
|
assert_eq!(.fields.tt, "13")
|
|
assert_eq!(.fields.termination_state, "----")
|
|
assert_eq!(.fields.retries, "0")
|
|
assert_eq!(.fields.status, "200")
|
|
assert_eq!(.fields.bytes, "512")
|
|
|
|
# --- Tier-1: glauth LDAP (VM, awaiting VM vector) ---
|
|
- name: glauth_routes_by_identifier
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.ldap1_syd1"
|
|
SYSLOG_IDENTIFIER: "glauth"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.glauth
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: glauth_parse_extracts_bind
|
|
inputs:
|
|
- insert_at: glauth_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.ldap1_syd1"
|
|
host: "ldap1"
|
|
SYSLOG_IDENTIFIER: "glauth"
|
|
message: '{"level":"info","msg":"Bind success as user","bindDN":"cn=admin,dc=example,dc=com","src":"192.0.2.40:1234","time":"2026-07-27T00:00:00Z"}'
|
|
outputs:
|
|
- extract_from: glauth_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.host, "ldap1")
|
|
assert_eq!(.severity, "info")
|
|
assert_eq!(.labels.app, "glauth")
|
|
assert_eq!(.fields.bindDN, "cn=admin,dc=example,dc=com")
|
|
assert_eq!(.fields.remote, "192.0.2.40:1234")
|
|
assert_eq!(.fields.success, "true")
|
|
|
|
# ================= Tier-2 (stacks on #318) =================
|
|
|
|
# --- BIND query logs (k8s bind-* + VM named) ---
|
|
- name: bind_routes_k8s_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.bind-internal.named"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.bind_query
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: bind_routes_vm_by_identifier
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.dns1_syd1"
|
|
SYSLOG_IDENTIFIER: "named"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.bind_query
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: bind_parse_extracts_query
|
|
inputs:
|
|
- insert_at: bind_query_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.bind-internal.named"
|
|
kubernetes.pod_namespace: "bind-internal"
|
|
kubernetes.container_name: "named"
|
|
kubernetes.pod_node_name: "node-4"
|
|
message: '02-Aug-2026 00:00:00.123 client @0x7f 192.0.2.1#40426 (www.example.com): view internal: query: www.example.com IN A +E(0)K (198.18.200.7)'
|
|
outputs:
|
|
- extract_from: bind_query_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "bind-internal")
|
|
assert_eq!(.labels.app, "bind")
|
|
assert_eq!(.message, "query www.example.com A")
|
|
assert_eq!(.fields.client_ip, "192.0.2.1")
|
|
assert_eq!(.fields.qname, "www.example.com")
|
|
assert_eq!(.fields.qclass, "IN")
|
|
assert_eq!(.fields.qtype, "A")
|
|
assert_eq!(.fields.view, "internal")
|
|
|
|
# --- Rancher audit (k8s, cattle-system sidecar) ---
|
|
- name: rancher_routes_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.cattle-system.rancher-audit-log"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.rancher_audit
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: rancher_parse_extracts_audit
|
|
inputs:
|
|
- insert_at: rancher_audit_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.cattle-system.rancher-audit-log"
|
|
kubernetes.pod_namespace: "cattle-system"
|
|
kubernetes.container_name: "rancher-audit-log"
|
|
kubernetes.pod_node_name: "node-5"
|
|
message: '{"auditID":"abc-123","requestURI":"/v3/tokens","user":{"name":"u-alice","group":["admins"]},"method":"GET","remoteAddr":"10.42.0.9:1234","responseCode":200,"requestTimestamp":"2026-08-01T00:00:00Z"}'
|
|
outputs:
|
|
- extract_from: rancher_audit_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "cattle-system")
|
|
assert_eq!(.labels.app, "rancher")
|
|
assert_eq!(.labels.log_type, "audit")
|
|
assert_eq!(.message, "GET /v3/tokens 200")
|
|
assert_eq!(.fields.user, "u-alice")
|
|
assert_eq!(.fields.verb, "GET")
|
|
assert_eq!(.fields.uri, "/v3/tokens")
|
|
assert_eq!(.fields.status, "200")
|
|
|
|
# --- CNPG Postgres (ONE transform, all clusters) ---
|
|
- name: cnpg_routes_by_postgres_container
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.litellm.postgres"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.cnpg_pg
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
# mutual exclusivity: an app-namespace CNPG pod (authentik) is claimed by
|
|
# cnpg_pg, NOT the authentik app route (which now carves out .postgres).
|
|
- name: cnpg_authentik_postgres_routes_to_cnpg
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.authentik.postgres"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.cnpg_pg
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: cnpg_parse_extracts_record
|
|
inputs:
|
|
- insert_at: cnpg_pg_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.litellm.postgres"
|
|
kubernetes.pod_namespace: "litellm"
|
|
kubernetes.container_name: "postgres"
|
|
kubernetes.pod_node_name: "node-6"
|
|
kubernetes.pod_labels."cnpg.io/cluster": "litellm-postgres"
|
|
message: '{"level":"info","ts":"2026-08-01T00:00:00Z","logger":"postgres","msg":"record","record":{"user_name":"litellm","database_name":"litellm","error_severity":"LOG","message":"duration: 12.345 ms statement: SELECT 1","query":""}}'
|
|
outputs:
|
|
- extract_from: cnpg_pg_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "litellm")
|
|
assert_eq!(.severity, "LOG")
|
|
assert_eq!(.labels.app, "cnpg")
|
|
assert_eq!(.labels.cluster, "litellm-postgres")
|
|
assert_eq!(.fields.error_severity, "LOG")
|
|
assert_eq!(.fields.duration_ms, "12.345")
|
|
assert_eq!(.fields.user, "litellm")
|
|
assert_eq!(.fields.database, "litellm")
|
|
|
|
# --- Gitea router/access (k8s + VM) ---
|
|
- name: gitea_routes_k8s_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.gitea.gitea"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.gitea
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: gitea_parse_router_line
|
|
inputs:
|
|
- insert_at: gitea_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.gitea.gitea"
|
|
kubernetes.pod_namespace: "gitea"
|
|
kubernetes.container_name: "gitea"
|
|
kubernetes.pod_node_name: "node-7"
|
|
message: '2026/08/01 00:00:00 .../router.go:100:func() [I] router: completed GET /user/login for 10.0.0.1:0, 200 OK in 12.3ms @ web/base.go:1'
|
|
outputs:
|
|
- extract_from: gitea_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "gitea")
|
|
assert_eq!(.labels.app, "gitea")
|
|
assert_eq!(.message, "GET /user/login 200")
|
|
assert_eq!(.fields.method, "GET")
|
|
assert_eq!(.fields.path, "/user/login")
|
|
assert_eq!(.fields.status, "200")
|
|
assert_eq!(.fields.latency, "12.3ms")
|
|
- name: gitea_parse_access_line
|
|
inputs:
|
|
- insert_at: gitea_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.gitea.gitea"
|
|
kubernetes.pod_namespace: "gitea"
|
|
kubernetes.container_name: "gitea"
|
|
message: '10.0.0.5 - alice [01/Aug/2026:00:00:00 +0000] "POST /repo/foo HTTP/1.1" 201 512 "-" "git/2.0"'
|
|
outputs:
|
|
- extract_from: gitea_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.fields.method, "POST")
|
|
assert_eq!(.fields.path, "/repo/foo")
|
|
assert_eq!(.fields.status, "201")
|
|
assert_eq!(.fields.user, "alice")
|
|
assert_eq!(.fields.client_ip, "10.0.0.5")
|
|
|
|
# --- PuppetServer / PuppetDB (k8s stdout) ---
|
|
- name: puppet_routes_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.puppet.puppetserver"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.puppet
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: puppet_parse_logback_line
|
|
inputs:
|
|
- insert_at: puppet_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.puppet.puppetserver"
|
|
kubernetes.pod_namespace: "puppet"
|
|
kubernetes.container_name: "puppetserver"
|
|
kubernetes.pod_node_name: "node-8"
|
|
message: '2026-08-01 00:00:00,123 INFO [qtp123-45] [puppetserver] Compiled catalog for web01.unkin.net in environment production in 1.23 seconds'
|
|
outputs:
|
|
- extract_from: puppet_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "puppet")
|
|
assert_eq!(.severity, "INFO")
|
|
assert_eq!(.labels.app, "puppet")
|
|
assert_eq!(.fields.level, "INFO")
|
|
assert_eq!(.fields.logger, "puppetserver")
|
|
assert_eq!(.fields.node, "web01.unkin.net")
|
|
|
|
# --- LiteLLM request logs (k8s JSON) ---
|
|
- name: litellm_routes_by_subject
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.litellm.litellm"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.litellm
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: litellm_parse_extracts_request
|
|
inputs:
|
|
- insert_at: litellm_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.k8s.litellm.litellm"
|
|
kubernetes.pod_namespace: "litellm"
|
|
kubernetes.container_name: "litellm"
|
|
kubernetes.pod_node_name: "node-9"
|
|
message: '{"message":"Request completed","level":"info","model":"gpt-4o","total_tokens":1234,"response_time":0.532,"api_key":"sk-abc","status":"success","timestamp":"2026-08-01T00:00:00Z"}'
|
|
outputs:
|
|
- extract_from: litellm_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "k8s")
|
|
assert_eq!(.namespace, "litellm")
|
|
assert_eq!(.severity, "info")
|
|
assert_eq!(.message, "Request completed")
|
|
assert_eq!(.labels.app, "litellm")
|
|
assert_eq!(.fields.model, "gpt-4o")
|
|
assert_eq!(.fields.tokens, "1234")
|
|
assert_eq!(.fields.latency, "0.532")
|
|
assert_eq!(.fields.key, "sk-abc")
|
|
assert_eq!(.fields.status, "success")
|
|
|
|
# --- Postfix maillog (VM, per-line best-effort) ---
|
|
- name: postfix_routes_by_identifier
|
|
inputs:
|
|
- insert_at: app_route
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.mail1_syd1"
|
|
SYSLOG_IDENTIFIER: "postfix/qmgr"
|
|
message: "routed"
|
|
outputs:
|
|
- extract_from: app_route.postfix
|
|
conditions:
|
|
- type: vrl
|
|
source: 'assert_eq!(.message, "routed")'
|
|
- name: postfix_parse_extracts_line
|
|
inputs:
|
|
- insert_at: postfix_parse
|
|
type: log
|
|
log_fields:
|
|
subject: "logs.vm.mail1_syd1"
|
|
host: "mail1"
|
|
SYSLOG_IDENTIFIER: "postfix/smtp"
|
|
message: 'ABC123DEF: to=<rcpt@example.com>, relay=mx.example.com[1.2.3.4]:25, delay=1.2, delays=0.1/0/0.5/0.6, dsn=2.0.0, status=sent (250 OK)'
|
|
outputs:
|
|
- extract_from: postfix_parse
|
|
conditions:
|
|
- type: vrl
|
|
source: |
|
|
assert_eq!(.source, "vm")
|
|
assert_eq!(.host, "mail1")
|
|
assert_eq!(.labels.app, "postfix")
|
|
assert_eq!(.fields.qid, "ABC123DEF")
|
|
assert_eq!(.fields.to, "rcpt@example.com")
|
|
assert_eq!(.fields.relay, "mx.example.com[1.2.3.4]:25")
|
|
assert_eq!(.fields.delay, "1.2")
|
|
assert_eq!(.fields.status, "sent")
|
|
assert_eq!(.fields.program, "postfix/smtp")
|