Size vlstorage for a 180d log store

- raise the vlstorage PVCs to 500Gi and the disk cap to 440GiB per node - move vlstorage onto cephrbd-fast-retain
Add a VictoriaLogs cluster and point logs-ingest at it
2026-09-26 19:02:34 +10:00 · 2026-09-26 18:37:47 +10:00
15 changed files with 301 additions and 163 deletions
@@ -20,3 +20,22 @@ spec:
    jsonData:
      timeInterval: "15s"
      httpMethod: "POST"
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: victorialogs
+  namespace: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  plugins:
+    - name: victoriametrics-logs-datasource
+      version: 0.32.0
+  datasource:
+    name: "VictoriaLogs"
+    type: "victoriametrics-logs-datasource"
+    uid: "victorialogs"
+    access: "proxy"
+    url: "http://vlselect-logs.logging.svc.cluster.local:9471"
@@ -11,6 +11,23 @@ data:
    /etc/haproxy/certs/main-unkin-net/tls.crt
    /etc/haproxy/certs/ceph-unkin-net/tls.crt

+  fe_http.map: |
+    sonarr.main.unkin.net be_sonarr
+    radarr.main.unkin.net be_radarr
+    lidarr.main.unkin.net be_lidarr
+    readarr.main.unkin.net be_readarr
+    prowlarr.main.unkin.net be_prowlarr
+    nzbget.main.unkin.net be_nzbget
+    jellyfin.main.unkin.net be_jellyfin
+    fafflix.unkin.net be_jellyfin
+    git.unkin.net be_gitea
+    grafana.unkin.net be_grafana
+    dashboard.ceph.unkin.net be_ceph_dashboard
+    mail-webadmin.main.unkin.net be_stalwart_webadmin
+    autoconfig.main.unkin.net be_stalwart_webadmin
+    autodiscovery.main.unkin.net be_stalwart_webadmin
+    auth.unkin.net be_k8s_kanidm
+
  fe_https.map: |
    sonarr.main.unkin.net be_sonarr
    radarr.main.unkin.net be_radarr
@@ -23,6 +40,9 @@ data:
    git.unkin.net be_gitea
    grafana.unkin.net be_grafana
    dashboard.ceph.unkin.net be_ceph_dashboard
+    mail-webadmin.main.unkin.net be_stalwart_webadmin
+    autoconfig.main.unkin.net be_stalwart_webadmin
+    autodiscovery.main.unkin.net be_stalwart_webadmin
    auth.unkin.net be_k8s_kanidm

  haproxy.cfg: |
@@ -59,16 +79,76 @@ data:
      timeout  http-keep-alive 10s
      timeout  check 10s

+    frontend fe_http
+      bind 0.0.0.0:80
+      mode http
+      description Global HTTP Frontend
+      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
+      http-request set-header X-Forwarded-Proto https
+      http-request set-header X-Real-IP %[src]
+      use_backend be_letsencrypt if acl-letsencrypt
+      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_http.map,be_default)]
+
    frontend fe_https
      bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
      mode http
      description Global HTTPS Frontend
+      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
+      acl acl_sonarr req.hdr(host) -i sonarr.main.unkin.net
+      acl acl_radarr req.hdr(host) -i radarr.main.unkin.net
+      acl acl_lidarr req.hdr(host) -i lidarr.main.unkin.net
+      acl acl_readarr req.hdr(host) -i readarr.main.unkin.net
+      acl acl_prowlarr req.hdr(host) -i prowlarr.main.unkin.net
+      acl acl_nzbget req.hdr(host) -i nzbget.main.unkin.net
+      acl acl_jellyfin req.hdr(host) -i jellyfin.main.unkin.net
+      acl acl_fafflix req.hdr(host) -i fafflix.unkin.net
+      acl acl_gitea req.hdr(host) -i git.unkin.net
+      acl acl_grafana req.hdr(host) -i grafana.unkin.net
+      acl acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net
+      acl acl_stalwart_webadmin req.hdr(host) -i mail-webadmin.main.unkin.net
+      acl acl_stalwart_webadmin req.hdr(host) -i autoconfig.main.unkin.net
+      acl acl_stalwart_webadmin req.hdr(host) -i autodiscovery.main.unkin.net
+      acl acl_kanidm req.hdr(host) -i auth.unkin.net
      http-request set-header X-Forwarded-Proto https
      http-request set-header X-Real-IP %[src]
+      http-response set-header X-Frame-Options DENY if acl_sonarr
+      http-response set-header X-Frame-Options DENY if acl_radarr
+      http-response set-header X-Frame-Options DENY if acl_lidarr
+      http-response set-header X-Frame-Options DENY if acl_readarr
+      http-response set-header X-Frame-Options DENY if acl_prowlarr
+      http-response set-header X-Frame-Options DENY if acl_nzbget
+      http-response set-header X-Frame-Options DENY if acl_jellyfin
+      http-response set-header X-Frame-Options DENY if acl_fafflix
+      http-response set-header X-Frame-Options DENY if acl_gitea
+      http-response set-header X-Frame-Options DENY if acl_grafana
+      http-response set-header X-Frame-Options DENY if acl_ceph_dashboard
+      http-response set-header X-Frame-Options DENY if acl_stalwart_webadmin
+      http-response set-header X-Frame-Options DENY if acl_kanidm
      http-response set-header X-Content-Type-Options nosniff
      http-response set-header X-XSS-Protection 1;mode=block
+      use_backend be_letsencrypt if acl-letsencrypt
      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]

+    frontend fe_imap
+      bind 0.0.0.0:143
+      mode tcp
+      description Frontend for Stalwart IMAP (STARTTLS)
+      default_backend be_stalwart_imap
+      log global
+      option tcplog
+      tcp-request inspect-delay 5s
+      tcp-request content accept if { req_len 0 }
+
+    frontend fe_imaps
+      bind 0.0.0.0:993
+      mode tcp
+      description Frontend for Stalwart IMAPS (implicit TLS)
+      default_backend be_stalwart_imaps
+      log global
+      option tcplog
+      tcp-request inspect-delay 5s
+      tcp-request content accept if { req_len 0 }
+
    frontend fe_metrics
      bind 0.0.0.0:8405
      mode http
@@ -77,6 +157,26 @@ data:
      http-request set-header X-Real-IP %[src]
      http-request use-service prometheus-exporter if { path /metrics }

+    frontend fe_smtp
+      bind 0.0.0.0:25
+      mode tcp
+      description Frontend for Stalwart SMTP
+      default_backend be_stalwart_smtp
+      log global
+      option tcplog
+      tcp-request inspect-delay 5s
+      tcp-request content accept if { req_len 0 }
+
+    frontend fe_submission
+      bind 0.0.0.0:587
+      mode tcp
+      description Frontend for Stalwart SMTP Submission
+      default_backend be_stalwart_submission
+      log global
+      option tcplog
+      tcp-request inspect-delay 5s
+      tcp-request content accept if { req_len 0 }
+
    backend be_ceph_dashboard
      description Backend for Ceph Dashboard from Mgr instances
      balance roundrobin
@@ -171,6 +271,11 @@ data:
      http-check expect status 200
      server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)

+    backend be_letsencrypt
+      description Backend for LetsEncrypt Verifications
+      balance roundrobin
+      server ausyd1nxvm2057 198.18.25.3:8888
+
    backend be_lidarr
      description Backend for au-syd1 lidarr
      balance roundrobin
@@ -255,12 +360,85 @@ data:
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none

+    backend be_stalwart_imap
+      description Backend for Stalwart IMAP (STARTTLS)
+      balance roundrobin
+      mode tcp
+      option tcp-check
+      option prefer-last-server
+      stick on src
+      stick-table type ip size 200k expire 30m
+      tcp-check connect port 143 send-proxy
+      tcp-check expect string "* OK"
+      tcp-check send "A001 STARTTLS\r\n"
+      tcp-check expect rstring "A001 (OK|2.0.0)"
+      server ausyd1nxvm2124 198.18.28.76:143 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2125 198.18.29.44:143 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2126 198.18.25.160:143 check fall 3 inter 3s rise 2 send-proxy-v2
+
+    backend be_stalwart_imaps
+      description Backend for Stalwart IMAPS (implicit TLS)
+      balance roundrobin
+      mode tcp
+      option tcp-check
+      option prefer-last-server
+      stick on src
+      stick-table type ip size 200k expire 30m
+      tcp-check connect ssl send-proxy
+      tcp-check expect string "* OK"
+      server ausyd1nxvm2124 198.18.28.76:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
+      server ausyd1nxvm2125 198.18.29.44:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
+      server ausyd1nxvm2126 198.18.25.160:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
+
+    backend be_stalwart_smtp
+      description Backend for Stalwart SMTP
+      balance roundrobin
+      mode tcp
+      option tcp-check
+      option prefer-last-server
+      stick on src
+      stick-table type ip size 200k expire 30m
+      tcp-check connect port 25 send-proxy
+      tcp-check expect string "220 "
+      server ausyd1nxvm2124 198.18.28.76:25 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2125 198.18.29.44:25 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2126 198.18.25.160:25 check fall 3 inter 3s rise 2 send-proxy-v2
+
+    backend be_stalwart_submission
+      description Backend for Stalwart SMTP Submission
+      balance roundrobin
+      mode tcp
+      option tcp-check
+      option prefer-last-server
+      stick on src
+      stick-table type ip size 200k expire 30m
+      tcp-check connect port 587 send-proxy
+      tcp-check expect string "220 "
+      server ausyd1nxvm2124 198.18.28.76:587 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2125 198.18.29.44:587 check fall 3 inter 3s rise 2 send-proxy-v2
+      server ausyd1nxvm2126 198.18.25.160:587 check fall 3 inter 3s rise 2 send-proxy-v2
+
+    backend be_stalwart_webadmin
+      description Backend for Stalwart Webadmin
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-check expect status 200
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 9443 }
+      http-reuse always
+      option httpchk GET /
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      stick-table type ip size 200k expire 30m
+      server ausyd1nxvm2124 198.18.28.76:443 check cookie ausyd1nxvm2124 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
+      server ausyd1nxvm2125 198.18.29.44:443 check cookie ausyd1nxvm2125 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
+      server ausyd1nxvm2126 198.18.25.160:443 check cookie ausyd1nxvm2126 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
+
    # The `peers au-syd1-prod` section is dropped: peer names must be static and a
-    # Deployment cannot provide them. Behind the external Traefik's TLS
-    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
-    # `stick on src` tables all key on that; the SRVNAME cookie carries real
-    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
-    # backend, so there is nothing to bind `accept-proxy` to.
+    # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a
+    # client to one replica so the per-replica stick-tables behave as before.

    listen health
      bind 0.0.0.0:8404
@@ -50,13 +50,27 @@ spec:
            readOnlyRootFilesystem: true
            capabilities:
              drop: [ALL]
-              # fe_https binds the privileged port 443 as uid 99, and the
-              # dst_port ACLs need the real port.
+              # Frontends bind 25/80/143/443/587; the dst_port ACLs need the real ports.
              add: [NET_BIND_SERVICE]
          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
            - name: https
              containerPort: 443
              protocol: TCP
+            - name: smtp
+              containerPort: 25
+              protocol: TCP
+            - name: imap
+              containerPort: 143
+              protocol: TCP
+            - name: submission
+              containerPort: 587
+              protocol: TCP
+            - name: imaps
+              containerPort: 993
+              protocol: TCP
            - name: health
              containerPort: 8404
              protocol: TCP
@@ -1,31 +0,0 @@
---
-# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
-# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
-# wildcard certs and terminates behind Traefik, so there are no certificateRefs
-# here. Listener hostnames are deliberately unset and the routes carry the
-# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
-# these listeners.
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    traefik.io/instance: external
-spec:
-  gatewayClassName: traefik-external
-  listeners:
-    - name: http
-      port: 80
-      protocol: HTTP
-      allowedRoutes:
-        namespaces:
-          from: Same
-    - name: https-passthrough
-      port: 443
-      protocol: TLS
-      tls:
-        mode: Passthrough
-      allowedRoutes:
-        namespaces:
-          from: Same
@@ -1,37 +0,0 @@
---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: haproxy-http-redirect
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - grafana.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
@@ -7,9 +7,6 @@ resources:
  - configmap.yaml
  - deployment.yaml
  - service.yaml
-  - gateway.yaml
-  - tlsroute.yaml
-  - httproute.yaml
  - pdb.yaml
  - vpa.yaml
  - vmpodscrape.yaml
@@ -4,16 +4,40 @@ kind: Service
 metadata:
  name: haproxy
  namespace: haproxy
+  annotations:
+    purelb.io/service-group: dmz
+    purelb.io/addresses: 198.18.199.1
 spec:
-  type: ClusterIP
-  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
-  # peer address here is a Traefik pod, not the client. sessionAffinity is
-  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
-  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
+  type: LoadBalancer
+  loadBalancerIP: "198.18.199.1"
+  # Source IP must survive for acl_internalsubnets, X-Real-IP and SMTP.
+  externalTrafficPolicy: Local
+  # Pins a client to one replica, standing in for the dropped stick-table peers.
+  sessionAffinity: ClientIP
  selector:
    app: haproxy
  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
    - name: https
      port: 443
      protocol: TCP
      targetPort: https
+    - name: smtp
+      port: 25
+      protocol: TCP
+      targetPort: smtp
+    - name: imap
+      port: 143
+      protocol: TCP
+      targetPort: imap
+    - name: submission
+      port: 587
+      protocol: TCP
+      targetPort: submission
+    - name: imaps
+      port: 993
+      protocol: TCP
+      targetPort: imaps
@@ -1,34 +0,0 @@
---
-apiVersion: gateway.networking.k8s.io/v1
-kind: TLSRoute
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - grafana.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: https-passthrough
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: haproxy
-          port: 443
-          weight: 1
@@ -1,16 +1,13 @@
 ---
-# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
-# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
-# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
-# resolve. The puppet-side Vector rollout ships NDJSON to
-# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
+# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client):
+# fronts the VLCluster vlinsert service over TLS at a name VMs can resolve.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
  name: logs-ingest
  namespace: logging
  labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
    app.kubernetes.io/component: ingest
    traefik.io/instance: internal
  annotations:
@@ -5,7 +5,7 @@ metadata:
  name: logs-ingest-http-redirect
  namespace: logging
  labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
    app.kubernetes.io/component: ingest
 spec:
  hostnames:
@@ -32,7 +32,7 @@ metadata:
  name: logs-ingest
  namespace: logging
  labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
    app.kubernetes.io/component: ingest
 spec:
  hostnames:
@@ -46,8 +46,8 @@ spec:
    - backendRefs:
        - group: ""
          kind: Service
-          name: vector-vm-ingest
-          port: 8080
+          name: vlinsert-logs
+          port: 9481
          weight: 1
      matches:
        - path:
@@ -10,6 +10,7 @@ resources:
  - job_clickhouse-schema.yaml
  - nats-bootstrap-job.yaml
  - cephrgw.yaml
+  - vlcluster.yaml
  - gateway.yaml
  - httproute.yaml
  - serviceaccount_logarchiver.yaml
@@ -0,0 +1,46 @@
+---
+apiVersion: operator.victoriametrics.com/v1
+kind: VLCluster
+metadata:
+  name: logs
+  namespace: logging
+spec:
+  vlinsert:
+    replicaCount: 2
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: "2"
+        memory: 4Gi
+  vlselect:
+    replicaCount: 2
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: "2"
+        memory: 4Gi
+  vlstorage:
+    replicaCount: 3
+    retentionPeriod: 180d
+    # ~7 GiB/day measured; 440GiB/node cap keeps 180d time-based, not disk-bound
+    retentionMaxDiskSpaceUsageBytes: 440GiB
+    storage:
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: cephrbd-fast-retain
+          resources:
+            requests:
+              storage: 500Gi
+    resources:
+      requests:
+        cpu: "1"
+        memory: 2Gi
+      limits:
+        cpu: "4"
+        memory: 8Gi
@@ -11,13 +11,11 @@ metadata:
  namespace: puppet
 spec:
  schedule: "*/1 * * * *"
-  startingDeadlineSeconds: 200
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
-      activeDeadlineSeconds: 300
      template:
        metadata:
          labels:
@@ -1,33 +0,0 @@
---
-# Shared Go build cache (GOCACHEPROG) for CI and developer laptops. Lives in the
-# woodpecker namespace because CI is the primary consumer and reads the Secret here.
-apiVersion: ceph.unkin.net/v1alpha1
-kind: ObjectStoreUser
-metadata:
-  name: gocache
-  namespace: woodpecker
-spec:
-  displayName: "Go build cache owner"
-  uid: gocache
-  maxBuckets: 1
-  secretName: gocache-s3
-  retainOnDelete: false
---
-apiVersion: ceph.unkin.net/v1alpha1
-kind: Bucket
-metadata:
-  name: gocache
-  namespace: woodpecker
-spec:
-  bucketName: gocache
-  ownerRef: gocache
-  versioning: false
-  # No placementTarget: default (replicated) placement, not the ec target the
-  # backup buckets use — a build cache is millions of small objects.
-  tags:
-    app: gocache
-    purpose: go-build-cache
-  retainOnDelete: false
-  # A cache bucket is never empty, and the operator refuses to delete a
-  # non-empty bucket without this, wedging the finalizer.
-  purgeOnDelete: true
@@ -7,7 +7,6 @@ resources:
  - cnpg_cluster.yaml
  - cnpg_backup.yaml
  - cnpg_pooler.yaml
-  - gocache_bucket.yaml
  - serviceaccount_arrproxy_ci.yaml
  - serviceaccount_autobackup_operator_ci.yaml
  - serviceaccount_ghp.yaml
Author	SHA1	Message	Date
unkin-agent	f193c7e75a	Size vlstorage for a 180d log store ci/woodpecker/pr/vector-test Pipeline was successful Details ci/woodpecker/pr/pre-commit Pipeline was successful Details ci/woodpecker/pr/kubeconform Pipeline was successful Details - raise the vlstorage PVCs to 500Gi and the disk cap to 440GiB per node - move vlstorage onto cephrbd-fast-retain	2026-09-26 19:02:34 +10:00
unkin-agent	efdfd3eb4b	Add a VictoriaLogs cluster and point logs-ingest at it ci/woodpecker/pr/vector-test Pipeline was successful Details ci/woodpecker/pr/pre-commit Pipeline was successful Details ci/woodpecker/pr/kubeconform Pipeline was successful Details - add VLCluster logs (2 vlinsert, 2 vlselect, 3 vlstorage, 180d) - repoint the logs-ingest HTTPRoute at vlinsert-logs:9481 - add a VictoriaLogs Grafana datasource and its plugin	2026-09-26 18:37:47 +10:00