feat: add victorialogs module (#374)

- add module for victorialogs
- add hieradata for vl insert/select/storage
- manage packages, directories, services, etc
- manage exporting metrics

Reviewed-on: #374

hieradata/roles/infra/logs/vlinsert.yaml

@@ -1,7 +1,50 @@
 ---
+hiera_include:
+  - victorialogs::node
+
+victorialogs::node::enable: true
+victorialogs::node::options:
+  httpListenAddr: ':9428'
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+  select.disable: 'undef'
+  storageNode:
+    - ausyd1nxvm2108.main.unkin.net:9428
+    - ausyd1nxvm2109.main.unkin.net:9428
+    - ausyd1nxvm2110.main.unkin.net:9428
+    - ausyd1nxvm2111.main.unkin.net:9428
+    - ausyd1nxvm2112.main.unkin.net:9428
+
 # additional altnames
 profiles::pki::vault::alt_names:
   - vlinsert.main.unkin.net
   - vlinsert.service.consul
   - vlinsert.query.consul
   - "vlinsert.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vlinsert:
+    service_name: 'vlinsert'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vlinsert'
+    address: "%{facts.networking.ip}"
+    port: 9428
+    checks:
+      - id: 'vlinsert_http_check'
+        name: 'vlinsert HTTP Check'
+        http: "https://%{facts.networking.fqdn}:9428/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vlinsert
+    disposition: write

hieradata/roles/infra/logs/vlselect.yaml

@@ -1,7 +1,50 @@
 ---
+hiera_include:
+  - victorialogs::node
+
+victorialogs::node::enable: true
+victorialogs::node::options:
+  httpListenAddr: ':9428'
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+  insert.disable: 'undef'
+  storageNode:
+    - ausyd1nxvm2108.main.unkin.net:9428
+    - ausyd1nxvm2109.main.unkin.net:9428
+    - ausyd1nxvm2110.main.unkin.net:9428
+    - ausyd1nxvm2111.main.unkin.net:9428
+    - ausyd1nxvm2112.main.unkin.net:9428
+
 # additional altnames
 profiles::pki::vault::alt_names:
   - vlselect.main.unkin.net
   - vlselect.service.consul
   - vlselect.query.consul
   - "vlselect.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vlselect:
+    service_name: 'vlselect'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vlselect'
+    address: "%{facts.networking.ip}"
+    port: 9428
+    checks:
+      - id: 'vlselect_http_check'
+        name: 'vlselect HTTP Check'
+        http: "https://%{facts.networking.fqdn}:9428/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vlselect
+    disposition: write

hieradata/roles/infra/logs/vlstorage.yaml

@@ -1,7 +1,45 @@
 ---
+hiera_include:
+  - victorialogs::node
+
+victorialogs::node::enable: true
+victorialogs::node::data_path: /data/vlstorage
+victorialogs::node::options:
+  storageDataPath: "%{hiera('victorialogs::node::data_path')}"
+  httpListenAddr: ':9428'
+  tls: 'true'
+  tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
+  tlsKeyFile: '/etc/pki/tls/vault/private.key'
+  tlsMinVersion: 'TLS12'
+  memory.allowedBytes: '1GiB'
+  metrics.exposeMetadata: 'true'
+  envflag.enable: 'true'
+
 # additional altnames
 profiles::pki::vault::alt_names:
   - vlstorage.main.unkin.net
   - vlstorage.service.consul
   - vlstorage.query.consul
   - "vlstorage.service.%{facts.country}-%{facts.region}.consul"
+
+consul::services:
+  vlstorage:
+    service_name: 'vlstorage'
+    tags:
+      - 'metrics'
+      - 'metrics_scheme=https'
+      - 'metrics_job=vlstorage'
+    address: "%{facts.networking.ip}"
+    port: 9428
+    checks:
+      - id: 'vlstorage_http_check'
+        name: 'vlstorage HTTP Check'
+        http: "https://%{facts.networking.fqdn}:9428/health"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: vlstorage
+    disposition: write

modules/victorialogs/manifests/node.pp

@@ -0,0 +1,79 @@
+class victorialogs::node (
+  Boolean $enable         = false,
+  String  $user           = 'victorialogs',
+  String  $group          = 'victorialogs',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Array[String] $packages = ['victoria-logs', 'vlutils'],
+  Stdlib::Absolutepath $exec_path = '/usr/bin/victoria-logs',
+  Stdlib::Absolutepath $data_path = '/var/lib/victorialogs',
+  Stdlib::Absolutepath $vars_file = '/etc/default/victoria-logs',
+  Hash[String, Variant[String, Array[String]]] $options = {},
+) {
+
+  # if enabled, manage this service
+  if $enable {
+
+    # install required packages
+    if $packages {
+      ensure_packages($packages, {ensure => 'installed'})
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage directories
+    file { [ $data_path ]:
+      ensure => directory,
+      owner  => $user,
+      group  => $group,
+    }
+
+    # manage environment options file
+    file { $vars_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('victorialogs/options.erb'),
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # manage the subscribed resources
+      if 'tls' in $options and $options['tls'] == 'true' {
+        if 'tlsCertFile' in $options and 'tlsKeyFile' in $options {
+          # tls option AND certs listed, subscribe to the options file and certs
+          $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]]
+        }else{
+          # tls option but no certs listed, just subscribe to the options file
+          warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from victorialogs options.')
+          $subscribe = [File[$vars_file]]
+        }
+      }else{
+        # no tls option, just subscribe to the options file
+        $subscribe = [File[$vars_file]]
+      }
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'victorialogs.service':
+        content   => template('victorialogs/victorialogs.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => $subscribe,
+      }
+    }
+  }
+}

modules/victorialogs/templates/options.erb

@@ -0,0 +1,33 @@
+<%
+  opts = @options || {}
+  lines = []
+
+  def undefish?(v)
+    v.respond_to?(:strip) && v.strip.downcase == 'undef'
+  end
+
+  opts.each do |key, val|
+    flag = "-#{key}"
+
+    case val
+    when Array
+      need_bare = false
+      val.each do |v|
+        if undefish?(v)
+          need_bare = true
+        else
+          lines << "#{flag}=#{v}"
+        end
+      end
+      lines << flag if need_bare
+    else
+      if undefish?(val)
+        lines << flag
+      else
+        lines << "#{flag}=#{val}"
+      end
+    end
+  end
+%>
+OPTIONS="<%= lines.join(" \\\n  ") %>"
+

modules/victorialogs/templates/victorialogs.service.erb

@@ -0,0 +1,18 @@
+[Unit]
+Description=VictoriaMetrics vminsert service
+After=network.target
+
+[Service]
+Type=simple
+User=<%= @user %>
+Group=<%= @group %>
+Restart=always
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %> $OPTIONS
+
+PrivateTmp=yes
+NoNewPrivileges=yes
+ProtectSystem=full
+
+[Install]
+WantedBy=multi-user.target