--- # `vector test` unit tests for the aggregator transforms. Merged with # aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the # per-app parsing follow-ups extend: add a test per new transform here. tests: - name: subject_routes_k8s_vs_vm inputs: - insert_at: route type: log log_fields: subject: "logs.k8s.shop.web" message: "routed" outputs: - extract_from: route.k8s conditions: - type: vrl source: | assert_eq!(.message, "routed") - name: k8s_log_is_normalised inputs: - insert_at: k8s_shape type: log log_fields: message: "hello from pod" stream: "stdout" timestamp: "2026-07-27T00:00:00Z" kubernetes.pod_name: "web-abc" kubernetes.pod_namespace: "shop" kubernetes.container_name: "web" kubernetes.pod_node_name: "node-1" outputs: - extract_from: k8s_shape conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "shop") assert_eq!(.pod, "web-abc") assert_eq!(.container, "web") assert_eq!(.host, "node-1") assert_eq!(.stream, "stdout") assert_eq!(.message, "hello from pod") - name: vm_log_is_normalised inputs: - insert_at: vm_shape type: log log_fields: message: "sshd started" host: "vm-db-1" severity: "info" role: "database" outputs: - extract_from: vm_shape conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.host, "vm-db-1") assert_eq!(.severity, "info") assert_eq!(.message, "sshd started") assert_eq!(.labels.role, "database") # --- catch-all preservation: an unclaimed k8s event still flows app_route -> # generic route -> k8s_shape (proves the two-stage chain keeps the fallback) --- - name: unclaimed_k8s_falls_through_to_generic inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.shop.web" message: "plain app log" outputs: - extract_from: route.k8s conditions: - type: vrl source: | assert_eq!(.message, "plain app log") # --- Tier-1: Authentik SSO (k8s, LIVE NOW) --- - name: authentik_routes_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.authentik.server" message: "routed" outputs: - extract_from: app_route.authentik conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: authentik_parse_extracts_event inputs: - insert_at: authentik_parse type: log log_fields: subject: "logs.k8s.authentik.server" stream: "stdout" kubernetes.pod_namespace: "authentik" kubernetes.container_name: "server" kubernetes.pod_node_name: "node-2" message: '{"event":"login","action":"login","user":"alice","client_ip":"203.0.113.9","result":"success","level":"info","logger":"authentik.events","timestamp":"2026-07-27T00:00:00Z"}' outputs: - extract_from: authentik_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "authentik") assert_eq!(.container, "server") assert_eq!(.severity, "info") assert_eq!(.message, "login") assert_eq!(.labels.app, "authentik") assert_eq!(.fields.event, "login") assert_eq!(.fields.action, "login") assert_eq!(.fields.user, "alice") assert_eq!(.fields.client_ip, "203.0.113.9") assert_eq!(.fields.result, "success") # --- Tier-1: Traefik ingress (k8s, JSON access logs) --- - name: traefik_routes_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.traefik-system.traefik" message: "routed" outputs: - extract_from: app_route.traefik conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: traefik_parse_extracts_access_fields inputs: - insert_at: traefik_parse type: log log_fields: subject: "logs.k8s.traefik-system.traefik" kubernetes.pod_namespace: "traefik-system" kubernetes.container_name: "traefik" kubernetes.pod_node_name: "node-3" message: '{"RouterName":"web@kubernetes","ServiceName":"shop-svc@kubernetes","RequestMethod":"GET","RequestPath":"/api","RequestHost":"shop.example.net","RequestProtocol":"HTTP/1.1","DownstreamStatus":200,"Duration":5000000,"ClientHost":"203.0.113.5","StartUTC":"2026-07-27T00:00:00Z"}' outputs: - extract_from: traefik_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "traefik-system") assert_eq!(.labels.app, "traefik") assert_eq!(.message, "GET /api 200") assert_eq!(.fields.route, "web@kubernetes") assert_eq!(.fields.service, "shop-svc@kubernetes") assert_eq!(.fields.method, "GET") assert_eq!(.fields.path, "/api") assert_eq!(.fields.host, "shop.example.net") assert_eq!(.fields.status, "200") assert_eq!(.fields.duration_ms, "5") assert_eq!(.fields.client_ip, "203.0.113.5") # --- Tier-1: Vault/OpenBao file audit (VM, awaiting VM vector) --- - name: vault_routes_by_file inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.vault1_syd1" file: "/var/log/vault_audit.log" message: "routed" outputs: - extract_from: app_route.vault conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: vault_parse_extracts_request inputs: - insert_at: vault_parse type: log log_fields: subject: "logs.vm.vault1_syd1" host: "vault1" file: "/var/log/vault_audit.log" message: '{"time":"2026-07-27T00:00:00Z","type":"response","auth":{"display_name":"token"},"request":{"operation":"read","path":"secret/data/app","remote_address":"10.0.0.9"},"error":""}' outputs: - extract_from: vault_parse conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.host, "vault1") assert_eq!(.labels.app, "vault") assert_eq!(.message, "read secret/data/app") assert_eq!(.fields.type, "response") assert_eq!(.fields.display_name, "token") assert_eq!(.fields.operation, "read") assert_eq!(.fields.path, "secret/data/app") assert_eq!(.fields.remote_address, "10.0.0.9") # --- Tier-1: nginx access (VM, awaiting VM vector) --- - name: nginx_access_routes_by_file inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.web1_syd1" file: "/var/log/nginx/shop_access.log" message: "routed" outputs: - extract_from: app_route.nginx_access conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: nginx_access_parse_extracts_combined inputs: - insert_at: nginx_access_parse type: log log_fields: subject: "logs.vm.web1_syd1" host: "web1" file: "/var/log/nginx/shop_access.log" message: '192.0.2.10 - - [27/Jul/2026:00:00:00 +0000] "GET /index.html HTTP/1.1" 200 1024 "https://ref.example/" "Mozilla/5.0" 0.012' outputs: - extract_from: nginx_access_parse conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.stream, "access") assert_eq!(.labels.log_type, "access") assert_eq!(.fields.client_ip, "192.0.2.10") assert_eq!(.fields.method, "GET") assert_eq!(.fields.path, "/index.html") assert_eq!(.fields.status, "200") assert_eq!(.fields.bytes, "1024") assert_eq!(.fields.referer, "https://ref.example/") assert_eq!(.fields.user_agent, "Mozilla/5.0") assert_eq!(.fields.request_time, "0.012") # --- Tier-1: nginx error (VM, awaiting VM vector) --- - name: nginx_error_routes_by_file inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.web1_syd1" file: "/var/log/nginx/shop_error.log" message: "routed" outputs: - extract_from: app_route.nginx_error conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: nginx_error_parse_extracts_fields inputs: - insert_at: nginx_error_parse type: log log_fields: subject: "logs.vm.web1_syd1" host: "web1" file: "/var/log/nginx/shop_error.log" message: '2026/07/27 00:00:00 [error] 1234#0: *5 open() "/var/www/x" failed (2: No such file or directory), client: 192.0.2.20, server: shop, request: "GET / HTTP/1.1", host: "shop"' outputs: - extract_from: nginx_error_parse conditions: - type: vrl source: | assert_eq!(.stream, "error") assert_eq!(.severity, "error") assert_eq!(.labels.log_type, "error") assert_eq!(.fields.level, "error") assert_eq!(.fields.pid, "1234") assert_eq!(.fields.cid, "5") assert_eq!(.fields.client_ip, "192.0.2.20") # --- Tier-1: HAProxy httplog (VM journald, awaiting VM vector) --- - name: haproxy_routes_by_identifier inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.halb1_syd1" SYSLOG_IDENTIFIER: "haproxy" message: "routed" outputs: - extract_from: app_route.haproxy conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: haproxy_parse_extracts_timers inputs: - insert_at: haproxy_parse type: log log_fields: subject: "logs.vm.halb1_syd1" host: "halb1" SYSLOG_IDENTIFIER: "haproxy" message: '192.0.2.30:54321 [27/Jul/2026:00:00:00.123] fe_http be_app/app1 10/0/1/2/13 200 512 - - ---- 5/4/3/2/0 0/0 "GET /health HTTP/1.1"' outputs: - extract_from: haproxy_parse conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.labels.app, "haproxy") assert_eq!(.fields.client_ip, "192.0.2.30") assert_eq!(.fields.frontend, "fe_http") assert_eq!(.fields.backend, "be_app") assert_eq!(.fields.server, "app1") assert_eq!(.fields.tq, "10") assert_eq!(.fields.tw, "0") assert_eq!(.fields.tc, "1") assert_eq!(.fields.tr, "2") assert_eq!(.fields.tt, "13") assert_eq!(.fields.termination_state, "----") assert_eq!(.fields.retries, "0") assert_eq!(.fields.status, "200") assert_eq!(.fields.bytes, "512") # --- Tier-1: glauth LDAP (VM, awaiting VM vector) --- - name: glauth_routes_by_identifier inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.ldap1_syd1" SYSLOG_IDENTIFIER: "glauth" message: "routed" outputs: - extract_from: app_route.glauth conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: glauth_parse_extracts_bind inputs: - insert_at: glauth_parse type: log log_fields: subject: "logs.vm.ldap1_syd1" host: "ldap1" SYSLOG_IDENTIFIER: "glauth" message: '{"level":"info","msg":"Bind success as user","bindDN":"cn=admin,dc=example,dc=com","src":"192.0.2.40:1234","time":"2026-07-27T00:00:00Z"}' outputs: - extract_from: glauth_parse conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.host, "ldap1") assert_eq!(.severity, "info") assert_eq!(.labels.app, "glauth") assert_eq!(.fields.bindDN, "cn=admin,dc=example,dc=com") assert_eq!(.fields.remote, "192.0.2.40:1234") assert_eq!(.fields.success, "true") # ================= Tier-2 (stacks on #318) ================= # --- BIND query logs (k8s bind-* + VM named) --- - name: bind_routes_k8s_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.bind-internal.named" message: "routed" outputs: - extract_from: app_route.bind_query conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: bind_routes_vm_by_identifier inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.dns1_syd1" SYSLOG_IDENTIFIER: "named" message: "routed" outputs: - extract_from: app_route.bind_query conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: bind_parse_extracts_query inputs: - insert_at: bind_query_parse type: log log_fields: subject: "logs.k8s.bind-internal.named" kubernetes.pod_namespace: "bind-internal" kubernetes.container_name: "named" kubernetes.pod_node_name: "node-4" message: '02-Aug-2026 00:00:00.123 client @0x7f 192.0.2.1#40426 (www.example.com): view internal: query: www.example.com IN A +E(0)K (198.18.200.7)' outputs: - extract_from: bind_query_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "bind-internal") assert_eq!(.labels.app, "bind") assert_eq!(.message, "query www.example.com A") assert_eq!(.fields.client_ip, "192.0.2.1") assert_eq!(.fields.qname, "www.example.com") assert_eq!(.fields.qclass, "IN") assert_eq!(.fields.qtype, "A") assert_eq!(.fields.view, "internal") # --- Rancher audit (k8s, cattle-system sidecar) --- - name: rancher_routes_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.cattle-system.rancher-audit-log" message: "routed" outputs: - extract_from: app_route.rancher_audit conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: rancher_parse_extracts_audit inputs: - insert_at: rancher_audit_parse type: log log_fields: subject: "logs.k8s.cattle-system.rancher-audit-log" kubernetes.pod_namespace: "cattle-system" kubernetes.container_name: "rancher-audit-log" kubernetes.pod_node_name: "node-5" message: '{"auditID":"abc-123","requestURI":"/v3/tokens","user":{"name":"u-alice","group":["admins"]},"method":"GET","remoteAddr":"10.42.0.9:1234","responseCode":200,"requestTimestamp":"2026-08-01T00:00:00Z"}' outputs: - extract_from: rancher_audit_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "cattle-system") assert_eq!(.labels.app, "rancher") assert_eq!(.labels.log_type, "audit") assert_eq!(.message, "GET /v3/tokens 200") assert_eq!(.fields.user, "u-alice") assert_eq!(.fields.verb, "GET") assert_eq!(.fields.uri, "/v3/tokens") assert_eq!(.fields.status, "200") # --- CNPG Postgres (ONE transform, all clusters) --- - name: cnpg_routes_by_postgres_container inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.litellm.postgres" message: "routed" outputs: - extract_from: app_route.cnpg_pg conditions: - type: vrl source: 'assert_eq!(.message, "routed")' # mutual exclusivity: an app-namespace CNPG pod (authentik) is claimed by # cnpg_pg, NOT the authentik app route (which now carves out .postgres). - name: cnpg_authentik_postgres_routes_to_cnpg inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.authentik.postgres" message: "routed" outputs: - extract_from: app_route.cnpg_pg conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: cnpg_parse_extracts_record inputs: - insert_at: cnpg_pg_parse type: log log_fields: subject: "logs.k8s.litellm.postgres" kubernetes.pod_namespace: "litellm" kubernetes.container_name: "postgres" kubernetes.pod_node_name: "node-6" kubernetes.pod_labels."cnpg.io/cluster": "litellm-postgres" message: '{"level":"info","ts":"2026-08-01T00:00:00Z","logger":"postgres","msg":"record","record":{"user_name":"litellm","database_name":"litellm","error_severity":"LOG","message":"duration: 12.345 ms statement: SELECT 1","query":""}}' outputs: - extract_from: cnpg_pg_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "litellm") assert_eq!(.severity, "LOG") assert_eq!(.labels.app, "cnpg") assert_eq!(.labels.cluster, "litellm-postgres") assert_eq!(.fields.error_severity, "LOG") assert_eq!(.fields.duration_ms, "12.345") assert_eq!(.fields.user, "litellm") assert_eq!(.fields.database, "litellm") # --- Gitea router/access (k8s + VM) --- - name: gitea_routes_k8s_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.gitea.gitea" message: "routed" outputs: - extract_from: app_route.gitea conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: gitea_parse_router_line inputs: - insert_at: gitea_parse type: log log_fields: subject: "logs.k8s.gitea.gitea" kubernetes.pod_namespace: "gitea" kubernetes.container_name: "gitea" kubernetes.pod_node_name: "node-7" message: '2026/08/01 00:00:00 .../router.go:100:func() [I] router: completed GET /user/login for 10.0.0.1:0, 200 OK in 12.3ms @ web/base.go:1' outputs: - extract_from: gitea_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "gitea") assert_eq!(.labels.app, "gitea") assert_eq!(.message, "GET /user/login 200") assert_eq!(.fields.method, "GET") assert_eq!(.fields.path, "/user/login") assert_eq!(.fields.status, "200") assert_eq!(.fields.latency, "12.3ms") - name: gitea_parse_access_line inputs: - insert_at: gitea_parse type: log log_fields: subject: "logs.k8s.gitea.gitea" kubernetes.pod_namespace: "gitea" kubernetes.container_name: "gitea" message: '10.0.0.5 - alice [01/Aug/2026:00:00:00 +0000] "POST /repo/foo HTTP/1.1" 201 512 "-" "git/2.0"' outputs: - extract_from: gitea_parse conditions: - type: vrl source: | assert_eq!(.fields.method, "POST") assert_eq!(.fields.path, "/repo/foo") assert_eq!(.fields.status, "201") assert_eq!(.fields.user, "alice") assert_eq!(.fields.client_ip, "10.0.0.5") # --- PuppetServer / PuppetDB (k8s stdout) --- - name: puppet_routes_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.puppet.puppetserver" message: "routed" outputs: - extract_from: app_route.puppet conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: puppet_parse_logback_line inputs: - insert_at: puppet_parse type: log log_fields: subject: "logs.k8s.puppet.puppetserver" kubernetes.pod_namespace: "puppet" kubernetes.container_name: "puppetserver" kubernetes.pod_node_name: "node-8" message: '2026-08-01 00:00:00,123 INFO [qtp123-45] [puppetserver] Compiled catalog for web01.unkin.net in environment production in 1.23 seconds' outputs: - extract_from: puppet_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "puppet") assert_eq!(.severity, "INFO") assert_eq!(.labels.app, "puppet") assert_eq!(.fields.level, "INFO") assert_eq!(.fields.logger, "puppetserver") assert_eq!(.fields.node, "web01.unkin.net") # --- LiteLLM request logs (k8s JSON) --- - name: litellm_routes_by_subject inputs: - insert_at: app_route type: log log_fields: subject: "logs.k8s.litellm.litellm" message: "routed" outputs: - extract_from: app_route.litellm conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: litellm_parse_extracts_request inputs: - insert_at: litellm_parse type: log log_fields: subject: "logs.k8s.litellm.litellm" kubernetes.pod_namespace: "litellm" kubernetes.container_name: "litellm" kubernetes.pod_node_name: "node-9" message: '{"message":"Request completed","level":"info","model":"gpt-4o","total_tokens":1234,"response_time":0.532,"api_key":"sk-abc","status":"success","timestamp":"2026-08-01T00:00:00Z"}' outputs: - extract_from: litellm_parse conditions: - type: vrl source: | assert_eq!(.source, "k8s") assert_eq!(.namespace, "litellm") assert_eq!(.severity, "info") assert_eq!(.message, "Request completed") assert_eq!(.labels.app, "litellm") assert_eq!(.fields.model, "gpt-4o") assert_eq!(.fields.tokens, "1234") assert_eq!(.fields.latency, "0.532") assert_eq!(.fields.key, "sk-abc") assert_eq!(.fields.status, "success") # --- Postfix maillog (VM, per-line best-effort) --- - name: postfix_routes_by_identifier inputs: - insert_at: app_route type: log log_fields: subject: "logs.vm.mail1_syd1" SYSLOG_IDENTIFIER: "postfix/qmgr" message: "routed" outputs: - extract_from: app_route.postfix conditions: - type: vrl source: 'assert_eq!(.message, "routed")' - name: postfix_parse_extracts_line inputs: - insert_at: postfix_parse type: log log_fields: subject: "logs.vm.mail1_syd1" host: "mail1" SYSLOG_IDENTIFIER: "postfix/smtp" message: 'ABC123DEF: to=, relay=mx.example.com[1.2.3.4]:25, delay=1.2, delays=0.1/0/0.5/0.6, dsn=2.0.0, status=sent (250 OK)' outputs: - extract_from: postfix_parse conditions: - type: vrl source: | assert_eq!(.source, "vm") assert_eq!(.host, "mail1") assert_eq!(.labels.app, "postfix") assert_eq!(.fields.qid, "ABC123DEF") assert_eq!(.fields.to, "rcpt@example.com") assert_eq!(.fields.relay, "mx.example.com[1.2.3.4]:25") assert_eq!(.fields.delay, "1.2") assert_eq!(.fields.status, "sent") assert_eq!(.fields.program, "postfix/smtp")