From efdfd3eb4badb03db1f306612607e1456a9adb8b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 18:37:47 +1000 Subject: [PATCH 1/5] Add a VictoriaLogs cluster and point logs-ingest at it - add VLCluster logs (2 vlinsert, 2 vlselect, 3 vlstorage, 180d) - repoint the logs-ingest HTTPRoute at vlinsert-logs:9481 - add a VictoriaLogs Grafana datasource and its plugin --- apps/base/grafana/grafanadatasource.yaml | 19 ++++++++++ apps/base/logging/gateway.yaml | 9 ++--- apps/base/logging/httproute.yaml | 8 ++-- apps/base/logging/kustomization.yaml | 1 + apps/base/logging/vlcluster.yaml | 47 ++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 apps/base/logging/vlcluster.yaml diff --git a/apps/base/grafana/grafanadatasource.yaml b/apps/base/grafana/grafanadatasource.yaml index 575a851..477aebf 100644 --- a/apps/base/grafana/grafanadatasource.yaml +++ b/apps/base/grafana/grafanadatasource.yaml @@ -20,3 +20,22 @@ spec: jsonData: timeInterval: "15s" httpMethod: "POST" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: victorialogs + namespace: grafana +spec: + instanceSelector: + matchLabels: + dashboards: "grafana" + plugins: + - name: victoriametrics-logs-datasource + version: 0.32.0 + datasource: + name: "VictoriaLogs" + type: "victoriametrics-logs-datasource" + uid: "victorialogs" + access: "proxy" + url: "http://vlselect-logs.logging.svc.cluster.local:9471" diff --git a/apps/base/logging/gateway.yaml b/apps/base/logging/gateway.yaml index bb072dc..67f01eb 100644 --- a/apps/base/logging/gateway.yaml +++ b/apps/base/logging/gateway.yaml @@ -1,16 +1,13 @@ --- -# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client). -# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so -# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can -# resolve. The puppet-side Vector rollout ships NDJSON to -# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task). +# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client): +# fronts the VLCluster vlinsert service over TLS at a name VMs can resolve. apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: logs-ingest namespace: logging labels: - app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/name: victorialogs app.kubernetes.io/component: ingest traefik.io/instance: internal annotations: diff --git a/apps/base/logging/httproute.yaml b/apps/base/logging/httproute.yaml index 2400863..cf390f1 100644 --- a/apps/base/logging/httproute.yaml +++ b/apps/base/logging/httproute.yaml @@ -5,7 +5,7 @@ metadata: name: logs-ingest-http-redirect namespace: logging labels: - app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/name: victorialogs app.kubernetes.io/component: ingest spec: hostnames: @@ -32,7 +32,7 @@ metadata: name: logs-ingest namespace: logging labels: - app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/name: victorialogs app.kubernetes.io/component: ingest spec: hostnames: @@ -46,8 +46,8 @@ spec: - backendRefs: - group: "" kind: Service - name: vector-vm-ingest - port: 8080 + name: vlinsert-logs + port: 9481 weight: 1 matches: - path: diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index 2464c25..cc3dc17 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -10,6 +10,7 @@ resources: - job_clickhouse-schema.yaml - nats-bootstrap-job.yaml - cephrgw.yaml + - vlcluster.yaml - gateway.yaml - httproute.yaml - serviceaccount_logarchiver.yaml diff --git a/apps/base/logging/vlcluster.yaml b/apps/base/logging/vlcluster.yaml new file mode 100644 index 0000000..fc3e88f --- /dev/null +++ b/apps/base/logging/vlcluster.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: operator.victoriametrics.com/v1 +kind: VLCluster +metadata: + name: logs + namespace: logging +spec: + vlinsert: + replicaCount: 2 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + vlselect: + replicaCount: 2 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + vlstorage: + replicaCount: 3 + retentionPeriod: 180d + # 3x250Gi is ~75% of the 180d estimate: the per-node disk cap drops the + # oldest data instead of filling the PVC, trading tail retention for Ceph. + retentionMaxDiskSpaceUsageBytes: 220GiB + storage: + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + storageClassName: cephrbd-fast-delete + resources: + requests: + storage: 250Gi + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "4" + memory: 8Gi -- 2.47.3 From f193c7e75aadd9e1dc9ae5776a1076ffbbd9bfd2 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 19:02:34 +1000 Subject: [PATCH 2/5] Size vlstorage for a 180d log store - raise the vlstorage PVCs to 500Gi and the disk cap to 440GiB per node - move vlstorage onto cephrbd-fast-retain --- apps/base/logging/vlcluster.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/base/logging/vlcluster.yaml b/apps/base/logging/vlcluster.yaml index fc3e88f..a8f1eaf 100644 --- a/apps/base/logging/vlcluster.yaml +++ b/apps/base/logging/vlcluster.yaml @@ -26,18 +26,17 @@ spec: vlstorage: replicaCount: 3 retentionPeriod: 180d - # 3x250Gi is ~75% of the 180d estimate: the per-node disk cap drops the - # oldest data instead of filling the PVC, trading tail retention for Ceph. - retentionMaxDiskSpaceUsageBytes: 220GiB + # ~7 GiB/day measured; 440GiB/node cap keeps 180d time-based, not disk-bound + retentionMaxDiskSpaceUsageBytes: 440GiB storage: volumeClaimTemplate: spec: accessModes: - ReadWriteOnce - storageClassName: cephrbd-fast-delete + storageClassName: cephrbd-fast-retain resources: requests: - storage: 250Gi + storage: 500Gi resources: requests: cpu: "1" -- 2.47.3 From acf074ca3caeaa9af3fb6fe092b7f8e8a8943cff Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 23:35:28 +1000 Subject: [PATCH 3/5] Size vlstorage at 250Gi per node on cephrbd-fast-delete --- apps/base/logging/vlcluster.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/base/logging/vlcluster.yaml b/apps/base/logging/vlcluster.yaml index a8f1eaf..80e9847 100644 --- a/apps/base/logging/vlcluster.yaml +++ b/apps/base/logging/vlcluster.yaml @@ -26,17 +26,17 @@ spec: vlstorage: replicaCount: 3 retentionPeriod: 180d - # ~7 GiB/day measured; 440GiB/node cap keeps 180d time-based, not disk-bound - retentionMaxDiskSpaceUsageBytes: 440GiB + # ~3 GiB/day measured; 220GiB/node cap keeps 180d time-based, not disk-bound + retentionMaxDiskSpaceUsageBytes: 220GiB storage: volumeClaimTemplate: spec: accessModes: - ReadWriteOnce - storageClassName: cephrbd-fast-retain + storageClassName: cephrbd-fast-delete resources: requests: - storage: 500Gi + storage: 250Gi resources: requests: cpu: "1" -- 2.47.3 From 74c092040ade02e6683939f6dab41de84d940d48 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 23:39:52 +1000 Subject: [PATCH 4/5] Pin VictoriaLogs cluster image version to v1.50.0 --- apps/base/logging/vlcluster.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/base/logging/vlcluster.yaml b/apps/base/logging/vlcluster.yaml index 80e9847..5f806ee 100644 --- a/apps/base/logging/vlcluster.yaml +++ b/apps/base/logging/vlcluster.yaml @@ -5,6 +5,7 @@ metadata: name: logs namespace: logging spec: + clusterVersion: v1.50.0 vlinsert: replicaCount: 2 resources: -- 2.47.3 From 6e0ac5fdbf38f6432e5c572b25352230ab78bbae Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 23:43:56 +1000 Subject: [PATCH 5/5] Set VictoriaLogs cluster image version to v1.52.0 --- apps/base/logging/vlcluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/base/logging/vlcluster.yaml b/apps/base/logging/vlcluster.yaml index 5f806ee..eb39a29 100644 --- a/apps/base/logging/vlcluster.yaml +++ b/apps/base/logging/vlcluster.yaml @@ -5,7 +5,7 @@ metadata: name: logs namespace: logging spec: - clusterVersion: v1.50.0 + clusterVersion: v1.52.0 vlinsert: replicaCount: 2 resources: -- 2.47.3