hieradata/roles/infra/git/server.yaml

@@ -38,7 +38,9 @@ consul::services:
     address: "%{facts.networking.ip}"
     port: 3000
     tags:
-      - metrics
+      - 'metrics'
+      - 'metrics_scheme=http'
+      - 'metrics_job=gitea'
     checks:
       - id: 'gitea_metrics_http_check'
         name: 'Gitea Metrics HTTP Check'

hieradata/roles/infra/halb/haproxy2.yaml

@@ -53,7 +53,8 @@ consul::services:
     service_name: 'haproxy-metrics'
     tags:
       - 'metrics'
-      - 'haproxy'
+      - 'metrics_scheme=https'
+      - 'metrics_job=haproxy'
     address: "%{facts.networking.ip}"
     port: 8405
     checks:

hieradata/roles/infra/metrics/grafana.yaml

@@ -29,7 +29,6 @@ consul::services:
     service_name: 'grafana'
     tags:
       - 'grafana'
-      - 'metrics'
     address: "%{facts.networking.ip}"
     port: 443
     checks:

hieradata/roles/infra/metrics/vmagent.yaml

@@ -1,4 +1,57 @@
 ---
-profiles::packages::include:
+hiera_include:
-  vmagent: {}
+  - vmcluster::vmagent
-  vmutils: {}
+
+vmcluster::vmagent::enable: true
+vmcluster::vmagent::options:
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  remoteWrite.url:
+    - https://vminsert.service.consul:8480/insert/0/prometheus
+
+  remoteWrite.tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  remoteWrite.tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  remoteWrite.tlsCAFile: '/etc/pki/tls/certs/ca-bundle.crt'
+  promscrape.config: '/etc/vmagent/scrape.yaml'
+  memory.allowedBytes: '1GiB'
+  envflag.enable: 'true'
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - vmagent.main.unkin.net
+  - vmagent.service.consul
+  - vmagent.query.consul
+  - "vmagent.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vmagent:
+    service_name: 'vmagent'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vmagent'
+    address: "%{facts.networking.ip}"
+    port: 8429
+    checks:
+      - id: 'vmagent_http_check'
+        name: 'vmagent HTTP Check'
+        http: "https://%{facts.networking.fqdn}:8429/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vmagent
+    disposition: write
+  - resource: agent_prefix
+    segment: ''
+    disposition: read
+  - resource: node_prefix
+    segment: ''
+    disposition: read
+  - resource: service_prefix
+    segment: ''
+    disposition: read

hieradata/roles/infra/metrics/vminsert.yaml

@@ -1,4 +1,50 @@
 ---
-profiles::packages::include:
+hiera_include:
-  vminsert: {}
+  - vmcluster::vminsert
-  vmutils: {}
+
+vmcluster::vminsert::enable: true
+vmcluster::vminsert::options:
+  replicationFactor: '2'
+  httpListenAddr: ':8480'
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+  storageNode:
+    - ausyd1nxvm2086.main.unkin.net:8400
+    - ausyd1nxvm2087.main.unkin.net:8400
+    - ausyd1nxvm2088.main.unkin.net:8400
+    - ausyd1nxvm2089.main.unkin.net:8400
+    - ausyd1nxvm2090.main.unkin.net:8400
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - vminsert.main.unkin.net
+  - vminsert.service.consul
+  - vminsert.query.consul
+  - "vminsert.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vminsert:
+    service_name: 'vminsert'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vminsert'
+    address: "%{facts.networking.ip}"
+    port: 8480
+    checks:
+      - id: 'vminsert_http_check'
+        name: 'vminsert HTTP Check'
+        http: "https://%{facts.networking.fqdn}:8480/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vminsert
+    disposition: write

hieradata/roles/infra/metrics/vmselect.yaml

@@ -1,4 +1,53 @@
 ---
-profiles::packages::include:
+hiera_include:
-  vmselect: {}
+  - vmcluster::vmselect
-  vmutils: {}
+
+vmcluster::vmselect::enable: true
+vmcluster::vmselect::data_path: /data/vmselect-cache
+vmcluster::vmselect::options:
+  search.skipSlowReplicas: 'true'
+  search.denyPartialResponse: 'false'
+  replicationFactor: '2'
+  globalReplicationFactor: '2'
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+  storageNode:
+    - ausyd1nxvm2086.main.unkin.net:8401
+    - ausyd1nxvm2087.main.unkin.net:8401
+    - ausyd1nxvm2088.main.unkin.net:8401
+    - ausyd1nxvm2089.main.unkin.net:8401
+    - ausyd1nxvm2090.main.unkin.net:8401
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - vmselect.main.unkin.net
+  - vmselect.service.consul
+  - vmselect.query.consul
+  - "vmselect.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vmselect:
+    service_name: 'vmselect'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vmselect'
+    address: "%{facts.networking.ip}"
+    port: 8481
+    checks:
+      - id: 'vmselect_http_check'
+        name: 'vmselect HTTP Check'
+        http: "https://%{facts.networking.fqdn}:8481/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vmselect
+    disposition: write

hieradata/roles/infra/metrics/vmstorage.yaml

@@ -1,4 +1,44 @@
 ---
-profiles::packages::include:
+hiera_include:
-  vmstorage: {}
+  - vmcluster::vmstorage
-  vmutils: {}
+
+vmcluster::vmstorage::enable: true
+vmcluster::vmstorage::data_path: /data/vmstorage
+vmcluster::vmstorage::options:
+  retentionPeriod: 90d
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - vmstorage.main.unkin.net
+  - vmstorage.service.consul
+  - vmstorage.query.consul
+  - "vmstorage.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vmstorage:
+    service_name: 'vmstorage'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vmstorage'
+    address: "%{facts.networking.ip}"
+    port: 8482
+    checks:
+      - id: 'vmstorage_http_check'
+        name: 'vmstorage HTTP Check'
+        http: "https://%{facts.networking.fqdn}:8482/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vmstorage
+    disposition: write

modules/libs/lib/facter/consul_node_token.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'puppet'
+
+Facter.add('consul_node_token') do
+  setcode do
+    token_file = '/root/.config/consul_node_token'
+    begin
+      File.read(token_file).strip
+    rescue StandardError
+      nil
+    end
+  end
+end

modules/vmcluster/manifests/vmagent.pp

@@ -0,0 +1,96 @@
+class vmcluster::vmagent (
+  Boolean $enable         = false,
+  String  $user           = 'vmagent',
+  String  $group          = 'vmagent',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Array[String] $packages = ['vmagent', 'vmutils'],
+  Stdlib::Absolutepath $exec_path = '/usr/bin/vmagent',
+  Stdlib::Absolutepath $data_path = '/var/lib/vmagent',
+  Stdlib::Absolutepath $vars_file = '/etc/default/vmagent',
+  String  $consul_node_token      = $facts['consul_node_token'],
+  Hash[String, Variant[String, Array[String]]] $options = {},
+) {
+
+  # if enabled, manage this service
+  if $enable {
+
+    # install required packages
+    if $packages {
+      ensure_packages($packages, {ensure => 'installed'})
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage directories
+    file { [ $data_path ]:
+      ensure => directory,
+      owner  => $user,
+      group  => $group,
+    }
+
+    # manage environment options file
+    file { $vars_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('vmcluster/options.erb'),
+    }
+
+    file { '/etc/vmagent':
+      ensure => directory,
+    }
+    file { '/etc/vmagent/scrape.yaml':
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('vmcluster/vmagent.scrape.yaml.erb'),
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # manage the subscribed resources
+      if 'tls' in $options and $options['tls'] == 'true' {
+        if 'remoteWrite.tlsCertFile' in $options and 'remoteWrite.tlsKeyFile' in $options {
+          # tls option AND certs listed, subscribe to the options file and certs
+          $subscribe = [
+            File[$vars_file],
+            File[$options['remoteWrite.tlsCertFile']],
+            File[$options['remoteWrite.tlsKeyFile']],
+            File['/etc/vmagent/scrape.yaml']
+          ]
+        }else{
+          # tls option but no certs listed, just subscribe to the options file
+          warning('TLS is enabled but remoteWrite.tlsCertFile or remoteWrite.tlsKeyFile is missing from vmagent options.')
+          $subscribe = [File[$vars_file], File['/etc/vmagent/scrape.yaml']]
+        }
+      }else{
+        # no tls option, just subscribe to the options file
+        $subscribe = [File[$vars_file], File['/etc/vmagent/scrape.yaml']]
+      }
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'vmagent.service':
+        content   => template('vmcluster/vmagent.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => $subscribe,
+      }
+    }
+  }
+}

modules/vmcluster/manifests/vminsert.pp

@@ -0,0 +1,71 @@
+class vmcluster::vminsert (
+  Boolean $enable         = false,
+  String  $user           = 'vminsert',
+  String  $group          = 'vminsert',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Array[String] $packages = ['vminsert', 'vmutils'],
+  Stdlib::Absolutepath $exec_path = '/usr/bin/vminsert',
+  Stdlib::Absolutepath $vars_file = '/etc/default/vminsert',
+  Hash[String, Variant[String, Array[String]]] $options = {},
+) {
+
+  # if enabled, manage this service
+  if $enable {
+
+    # install required packages
+    if $packages {
+      ensure_packages($packages, {ensure => 'installed'})
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage environment options file
+    file { $vars_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('vmcluster/options.erb'),
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # manage the subscribed resources
+      if 'tls' in $options and $options['tls'] == 'true' {
+        if 'tlsCertFile' in $options and 'tlsKeyFile' in $options {
+          # tls option AND certs listed, subscribe to the options file and certs
+          $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]]
+        }else{
+          # tls option but no certs listed, just subscribe to the options file
+          warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vminsert options.')
+          $subscribe = [File[$vars_file]]
+        }
+      }else{
+        # no tls option, just subscribe to the options file
+        $subscribe = [File[$vars_file]]
+      }
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'vminsert.service':
+        content   => template('vmcluster/vminsert.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => $subscribe,
+      }
+    }
+  }
+}

modules/vmcluster/manifests/vmselect.pp

@@ -0,0 +1,79 @@
+class vmcluster::vmselect (
+  Boolean $enable         = false,
+  String  $user           = 'vmselect',
+  String  $group          = 'vmselect',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Array[String] $packages = ['vmselect', 'vmutils'],
+  Stdlib::Absolutepath $exec_path = '/usr/bin/vmselect',
+  Stdlib::Absolutepath $data_path = '/var/lib/vmselect',
+  Stdlib::Absolutepath $vars_file = '/etc/default/vmselect',
+  Hash[String, Variant[String, Array[String]]] $options = {},
+) {
+
+  # if enabled, manage this service
+  if $enable {
+
+    # install required packages
+    if $packages {
+      ensure_packages($packages, {ensure => 'installed'})
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage directories
+    file { [ $data_path ]:
+      ensure => directory,
+      owner  => $user,
+      group  => $group,
+    }
+
+    # manage environment options file
+    file { $vars_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('vmcluster/options.erb'),
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # manage the subscribed resources
+      if 'tls' in $options and $options['tls'] == 'true' {
+        if 'tlsCertFile' in $options and 'tlsKeyFile' in $options {
+          # tls option AND certs listed, subscribe to the options file and certs
+          $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]]
+        }else{
+          # tls option but no certs listed, just subscribe to the options file
+          warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vmselect options.')
+          $subscribe = [File[$vars_file]]
+        }
+      }else{
+        # no tls option, just subscribe to the options file
+        $subscribe = [File[$vars_file]]
+      }
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'vmselect.service':
+        content   => template('vmcluster/vmselect.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => $subscribe,
+      }
+    }
+  }
+}

modules/vmcluster/manifests/vmstorage.pp

@@ -0,0 +1,79 @@
+class vmcluster::vmstorage (
+  Boolean $enable         = false,
+  String  $user           = 'vmstorage',
+  String  $group          = 'vmstorage',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Array[String] $packages = ['vmstorage', 'vmutils'],
+  Stdlib::Absolutepath $exec_path = '/usr/bin/vmstorage',
+  Stdlib::Absolutepath $data_path = '/var/lib/vmstorage',
+  Stdlib::Absolutepath $vars_file = '/etc/default/vmstorage',
+  Hash[String, Variant[String, Array[String]]] $options = {},
+) {
+
+  # if enabled, manage this service
+  if $enable {
+
+    # install required packages
+    if $packages {
+      ensure_packages($packages, {ensure => 'installed'})
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage directories
+    file { [ $data_path ]:
+      ensure => directory,
+      owner  => $user,
+      group  => $group,
+    }
+
+    # manage environment options file
+    file { $vars_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('vmcluster/options.erb'),
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # manage the subscribed resources
+      if 'tls' in $options and $options['tls'] == 'true' {
+        if 'tlsCertFile' in $options and 'tlsKeyFile' in $options {
+          # tls option AND certs listed, subscribe to the options file and certs
+          $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]]
+        }else{
+          # tls option but no certs listed, just subscribe to the options file
+          warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vmstorage options.')
+          $subscribe = [File[$vars_file]]
+        }
+      }else{
+        # no tls option, just subscribe to the options file
+        $subscribe = [File[$vars_file]]
+      }
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'vmstorage.service':
+        content   => template('vmcluster/vmstorage.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => $subscribe,
+      }
+    }
+  }
+}

modules/vmcluster/templates/options.erb

@@ -0,0 +1,18 @@
+<%
+  opts = @options || {}
+  lines = []
+
+  opts.each do |key, val|
+    flag = "-#{key}"
+
+    if val.is_a?(Array)
+      val.each do |v|
+        lines << "#{flag}=#{v}"
+      end
+    else
+      lines << "#{flag}=#{val}"
+    end
+  end
+%>
+OPTIONS="<%= lines.map { |l| l.lstrip }.join(" \\\n  ") %>"
+

modules/vmcluster/templates/vmagent.scrape.yaml.erb

@@ -0,0 +1,37 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+
+scrape_configs:
+  - job_name: 'consul'
+    consul_sd_configs:
+      - server: 'consul.service.consul:8500'
+        token: <%= @consul_node_token %>
+        scheme: http
+
+    relabel_configs:
+      # Only keep services with the `metrics` tag
+      - source_labels: [__meta_consul_tagpresent_metrics]
+        regex: true
+        action: keep
+
+      # Use the service address and port
+      - source_labels: [__meta_consul_node, __meta_consul_service_port]
+        separator: ':'
+        target_label: __address__
+        replacement: '${1}:${2}'
+        action: replace
+
+      # Set scheme to metrics_scheme
+      - source_labels: [__meta_consul_tag_metrics_scheme]
+        target_label: __scheme__
+        action: replace
+
+      # Set path to /metrics
+      - target_label: __metrics_path__
+        replacement: /metrics
+
+      # Use metrics_job=<job> as job label
+      - source_labels: [__meta_consul_tag_metrics_job]
+        target_label: job
+        action: replace

modules/vmcluster/templates/vmagent.service.erb

@@ -0,0 +1,18 @@
+[Unit]
+Description=VictoriaMetrics vmagent service
+After=network.target
+
+[Service]
+Type=simple
+User=<%= @user %>
+Group=<%= @group %>
+Restart=always
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %> -remoteWrite.tmpDataPath=<%= @data_path %> $OPTIONS
+
+PrivateTmp=yes
+NoNewPrivileges=yes
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target

modules/vmcluster/templates/vminsert.service.erb

@@ -0,0 +1,18 @@
+[Unit]
+Description=VictoriaMetrics vminsert service
+After=network.target
+
+[Service]
+Type=simple
+User=<%= @user %>
+Group=<%= @group %>
+Restart=always
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %> $OPTIONS
+
+PrivateTmp=yes
+NoNewPrivileges=yes
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target

modules/vmcluster/templates/vmselect.service.erb

@@ -0,0 +1,18 @@
+[Unit]
+Description=VictoriaMetrics vmselect service
+After=network.target
+
+[Service]
+Type=simple
+User=<%= @user %>
+Group=<%= @group %>
+Restart=always
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %> -cacheDataPath=<%= @data_path %> $OPTIONS
+
+PrivateTmp=yes
+NoNewPrivileges=yes
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target

modules/vmcluster/templates/vmstorage.service.erb

@@ -0,0 +1,18 @@
+[Unit]
+Description=VictoriaMetrics vmstorage service
+After=network.target
+
+[Service]
+Type=simple
+User=<%= @user %>
+Group=<%= @group %>
+Restart=always
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %> -storageDataPath=<%= @data_path %> $OPTIONS
+
+PrivateTmp=yes
+NoNewPrivileges=yes
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target