diff --git a/hieradata/roles/infra/git/server.yaml b/hieradata/roles/infra/git/server.yaml index 208c6f4..6319a8d 100644 --- a/hieradata/roles/infra/git/server.yaml +++ b/hieradata/roles/infra/git/server.yaml @@ -38,7 +38,9 @@ consul::services: address: "%{facts.networking.ip}" port: 3000 tags: - - metrics + - 'metrics' + - 'metrics_scheme=http' + - 'metrics_job=gitea' checks: - id: 'gitea_metrics_http_check' name: 'Gitea Metrics HTTP Check' diff --git a/hieradata/roles/infra/halb/haproxy2.yaml b/hieradata/roles/infra/halb/haproxy2.yaml index 947015f..98e74f9 100644 --- a/hieradata/roles/infra/halb/haproxy2.yaml +++ b/hieradata/roles/infra/halb/haproxy2.yaml @@ -53,7 +53,8 @@ consul::services: service_name: 'haproxy-metrics' tags: - 'metrics' - - 'haproxy' + - 'metrics_scheme=https' + - 'metrics_job=haproxy' address: "%{facts.networking.ip}" port: 8405 checks: diff --git a/hieradata/roles/infra/metrics/grafana.yaml b/hieradata/roles/infra/metrics/grafana.yaml index 6f81f06..11b1c3c 100644 --- a/hieradata/roles/infra/metrics/grafana.yaml +++ b/hieradata/roles/infra/metrics/grafana.yaml @@ -29,7 +29,6 @@ consul::services: service_name: 'grafana' tags: - 'grafana' - - 'metrics' address: "%{facts.networking.ip}" port: 443 checks: diff --git a/hieradata/roles/infra/metrics/vmagent.yaml b/hieradata/roles/infra/metrics/vmagent.yaml index 55940e6..5db56e6 100644 --- a/hieradata/roles/infra/metrics/vmagent.yaml +++ b/hieradata/roles/infra/metrics/vmagent.yaml @@ -1,4 +1,57 @@ --- -profiles::packages::include: - vmagent: {} - vmutils: {} +hiera_include: + - vmcluster::vmagent + +vmcluster::vmagent::enable: true +vmcluster::vmagent::options: + tls: 'true' + tlsCertFile: '/etc/pki/tls/vault/certificate.crt' + tlsKeyFile: '/etc/pki/tls/vault/private.key' + tlsMinVersion: 'TLS12' + remoteWrite.url: + - https://vminsert.service.consul:8480/insert/0/prometheus + + remoteWrite.tlsCertFile: '/etc/pki/tls/vault/certificate.crt' + remoteWrite.tlsKeyFile: '/etc/pki/tls/vault/private.key' + remoteWrite.tlsCAFile: '/etc/pki/tls/certs/ca-bundle.crt' + promscrape.config: '/etc/vmagent/scrape.yaml' + memory.allowedBytes: '1GiB' + envflag.enable: 'true' + +# additional altnames +profiles::pki::vault::alt_names: + - vmagent.main.unkin.net + - vmagent.service.consul + - vmagent.query.consul + - "vmagent.service.%{facts.country}-%{facts.region}.consul" + +consul::services: + vmagent: + service_name: 'vmagent' + tags: + - 'metrics' + - 'metrics_scheme=https' + - 'metrics_job=vmagent' + address: "%{facts.networking.ip}" + port: 8429 + checks: + - id: 'vmagent_http_check' + name: 'vmagent HTTP Check' + http: "https://%{facts.networking.fqdn}:8429/health" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: vmagent + disposition: write + - resource: agent_prefix + segment: '' + disposition: read + - resource: node_prefix + segment: '' + disposition: read + - resource: service_prefix + segment: '' + disposition: read diff --git a/hieradata/roles/infra/metrics/vminsert.yaml b/hieradata/roles/infra/metrics/vminsert.yaml index 09042ee..ac5869e 100644 --- a/hieradata/roles/infra/metrics/vminsert.yaml +++ b/hieradata/roles/infra/metrics/vminsert.yaml @@ -1,4 +1,50 @@ --- -profiles::packages::include: - vminsert: {} - vmutils: {} +hiera_include: + - vmcluster::vminsert + +vmcluster::vminsert::enable: true +vmcluster::vminsert::options: + replicationFactor: '2' + httpListenAddr: ':8480' + tls: 'true' + tlsCertFile: '/etc/pki/tls/vault/certificate.crt' + tlsKeyFile: '/etc/pki/tls/vault/private.key' + tlsMinVersion: 'TLS12' + memory.allowedBytes: '1GiB' + metrics.exposeMetadata: 'true' + envflag.enable: 'true' + storageNode: + - ausyd1nxvm2086.main.unkin.net:8400 + - ausyd1nxvm2087.main.unkin.net:8400 + - ausyd1nxvm2088.main.unkin.net:8400 + - ausyd1nxvm2089.main.unkin.net:8400 + - ausyd1nxvm2090.main.unkin.net:8400 + +# additional altnames +profiles::pki::vault::alt_names: + - vminsert.main.unkin.net + - vminsert.service.consul + - vminsert.query.consul + - "vminsert.service.%{facts.country}-%{facts.region}.consul" + +consul::services: + vminsert: + service_name: 'vminsert' + tags: + - 'metrics' + - 'metrics_scheme=https' + - 'metrics_job=vminsert' + address: "%{facts.networking.ip}" + port: 8480 + checks: + - id: 'vminsert_http_check' + name: 'vminsert HTTP Check' + http: "https://%{facts.networking.fqdn}:8480/health" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: vminsert + disposition: write diff --git a/hieradata/roles/infra/metrics/vmselect.yaml b/hieradata/roles/infra/metrics/vmselect.yaml index 26abd40..aea211f 100644 --- a/hieradata/roles/infra/metrics/vmselect.yaml +++ b/hieradata/roles/infra/metrics/vmselect.yaml @@ -1,4 +1,53 @@ --- -profiles::packages::include: - vmselect: {} - vmutils: {} +hiera_include: + - vmcluster::vmselect + +vmcluster::vmselect::enable: true +vmcluster::vmselect::data_path: /data/vmselect-cache +vmcluster::vmselect::options: + search.skipSlowReplicas: 'true' + search.denyPartialResponse: 'false' + replicationFactor: '2' + globalReplicationFactor: '2' + tls: 'true' + tlsCertFile: '/etc/pki/tls/vault/certificate.crt' + tlsKeyFile: '/etc/pki/tls/vault/private.key' + tlsMinVersion: 'TLS12' + memory.allowedBytes: '1GiB' + metrics.exposeMetadata: 'true' + envflag.enable: 'true' + storageNode: + - ausyd1nxvm2086.main.unkin.net:8401 + - ausyd1nxvm2087.main.unkin.net:8401 + - ausyd1nxvm2088.main.unkin.net:8401 + - ausyd1nxvm2089.main.unkin.net:8401 + - ausyd1nxvm2090.main.unkin.net:8401 + +# additional altnames +profiles::pki::vault::alt_names: + - vmselect.main.unkin.net + - vmselect.service.consul + - vmselect.query.consul + - "vmselect.service.%{facts.country}-%{facts.region}.consul" + +consul::services: + vmselect: + service_name: 'vmselect' + tags: + - 'metrics' + - 'metrics_scheme=https' + - 'metrics_job=vmselect' + address: "%{facts.networking.ip}" + port: 8481 + checks: + - id: 'vmselect_http_check' + name: 'vmselect HTTP Check' + http: "https://%{facts.networking.fqdn}:8481/health" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: vmselect + disposition: write diff --git a/hieradata/roles/infra/metrics/vmstorage.yaml b/hieradata/roles/infra/metrics/vmstorage.yaml index a005592..ca84896 100644 --- a/hieradata/roles/infra/metrics/vmstorage.yaml +++ b/hieradata/roles/infra/metrics/vmstorage.yaml @@ -1,4 +1,44 @@ --- -profiles::packages::include: - vmstorage: {} - vmutils: {} +hiera_include: + - vmcluster::vmstorage + +vmcluster::vmstorage::enable: true +vmcluster::vmstorage::data_path: /data/vmstorage +vmcluster::vmstorage::options: + retentionPeriod: 90d + tls: 'true' + tlsCertFile: '/etc/pki/tls/vault/certificate.crt' + tlsKeyFile: '/etc/pki/tls/vault/private.key' + tlsMinVersion: 'TLS12' + memory.allowedBytes: '1GiB' + metrics.exposeMetadata: 'true' + envflag.enable: 'true' + +# additional altnames +profiles::pki::vault::alt_names: + - vmstorage.main.unkin.net + - vmstorage.service.consul + - vmstorage.query.consul + - "vmstorage.service.%{facts.country}-%{facts.region}.consul" + +consul::services: + vmstorage: + service_name: 'vmstorage' + tags: + - 'metrics' + - 'metrics_scheme=https' + - 'metrics_job=vmstorage' + address: "%{facts.networking.ip}" + port: 8482 + checks: + - id: 'vmstorage_http_check' + name: 'vmstorage HTTP Check' + http: "https://%{facts.networking.fqdn}:8482/health" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: vmstorage + disposition: write diff --git a/modules/libs/lib/facter/consul_node_token.rb b/modules/libs/lib/facter/consul_node_token.rb new file mode 100644 index 0000000..d609ecf --- /dev/null +++ b/modules/libs/lib/facter/consul_node_token.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'puppet' + +Facter.add('consul_node_token') do + setcode do + token_file = '/root/.config/consul_node_token' + begin + File.read(token_file).strip + rescue StandardError + nil + end + end +end diff --git a/modules/vmcluster/manifests/vmagent.pp b/modules/vmcluster/manifests/vmagent.pp new file mode 100644 index 0000000..3a02703 --- /dev/null +++ b/modules/vmcluster/manifests/vmagent.pp @@ -0,0 +1,96 @@ +class vmcluster::vmagent ( + Boolean $enable = false, + String $user = 'vmagent', + String $group = 'vmagent', + Boolean $manage_user = true, + Boolean $manage_service = true, + Array[String] $packages = ['vmagent', 'vmutils'], + Stdlib::Absolutepath $exec_path = '/usr/bin/vmagent', + Stdlib::Absolutepath $data_path = '/var/lib/vmagent', + Stdlib::Absolutepath $vars_file = '/etc/default/vmagent', + String $consul_node_token = $facts['consul_node_token'], + Hash[String, Variant[String, Array[String]]] $options = {}, +) { + + # if enabled, manage this service + if $enable { + + # install required packages + if $packages { + ensure_packages($packages, {ensure => 'installed'}) + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage directories + file { [ $data_path ]: + ensure => directory, + owner => $user, + group => $group, + } + + # manage environment options file + file { $vars_file: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('vmcluster/options.erb'), + } + + file { '/etc/vmagent': + ensure => directory, + } + file { '/etc/vmagent/scrape.yaml': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('vmcluster/vmagent.scrape.yaml.erb'), + } + + # manage the systemd service + if $manage_service { + + # manage the subscribed resources + if 'tls' in $options and $options['tls'] == 'true' { + if 'remoteWrite.tlsCertFile' in $options and 'remoteWrite.tlsKeyFile' in $options { + # tls option AND certs listed, subscribe to the options file and certs + $subscribe = [ + File[$vars_file], + File[$options['remoteWrite.tlsCertFile']], + File[$options['remoteWrite.tlsKeyFile']], + File['/etc/vmagent/scrape.yaml'] + ] + }else{ + # tls option but no certs listed, just subscribe to the options file + warning('TLS is enabled but remoteWrite.tlsCertFile or remoteWrite.tlsKeyFile is missing from vmagent options.') + $subscribe = [File[$vars_file], File['/etc/vmagent/scrape.yaml']] + } + }else{ + # no tls option, just subscribe to the options file + $subscribe = [File[$vars_file], File['/etc/vmagent/scrape.yaml']] + } + + # Use these in notifications or file resources + systemd::unit_file { 'vmagent.service': + content => template('vmcluster/vmagent.service.erb'), + enable => true, + active => true, + subscribe => $subscribe, + } + } + } +} diff --git a/modules/vmcluster/manifests/vminsert.pp b/modules/vmcluster/manifests/vminsert.pp new file mode 100644 index 0000000..a85742c --- /dev/null +++ b/modules/vmcluster/manifests/vminsert.pp @@ -0,0 +1,71 @@ +class vmcluster::vminsert ( + Boolean $enable = false, + String $user = 'vminsert', + String $group = 'vminsert', + Boolean $manage_user = true, + Boolean $manage_service = true, + Array[String] $packages = ['vminsert', 'vmutils'], + Stdlib::Absolutepath $exec_path = '/usr/bin/vminsert', + Stdlib::Absolutepath $vars_file = '/etc/default/vminsert', + Hash[String, Variant[String, Array[String]]] $options = {}, +) { + + # if enabled, manage this service + if $enable { + + # install required packages + if $packages { + ensure_packages($packages, {ensure => 'installed'}) + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage environment options file + file { $vars_file: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('vmcluster/options.erb'), + } + + # manage the systemd service + if $manage_service { + + # manage the subscribed resources + if 'tls' in $options and $options['tls'] == 'true' { + if 'tlsCertFile' in $options and 'tlsKeyFile' in $options { + # tls option AND certs listed, subscribe to the options file and certs + $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]] + }else{ + # tls option but no certs listed, just subscribe to the options file + warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vminsert options.') + $subscribe = [File[$vars_file]] + } + }else{ + # no tls option, just subscribe to the options file + $subscribe = [File[$vars_file]] + } + + # Use these in notifications or file resources + systemd::unit_file { 'vminsert.service': + content => template('vmcluster/vminsert.service.erb'), + enable => true, + active => true, + subscribe => $subscribe, + } + } + } +} diff --git a/modules/vmcluster/manifests/vmselect.pp b/modules/vmcluster/manifests/vmselect.pp new file mode 100644 index 0000000..8cac961 --- /dev/null +++ b/modules/vmcluster/manifests/vmselect.pp @@ -0,0 +1,79 @@ +class vmcluster::vmselect ( + Boolean $enable = false, + String $user = 'vmselect', + String $group = 'vmselect', + Boolean $manage_user = true, + Boolean $manage_service = true, + Array[String] $packages = ['vmselect', 'vmutils'], + Stdlib::Absolutepath $exec_path = '/usr/bin/vmselect', + Stdlib::Absolutepath $data_path = '/var/lib/vmselect', + Stdlib::Absolutepath $vars_file = '/etc/default/vmselect', + Hash[String, Variant[String, Array[String]]] $options = {}, +) { + + # if enabled, manage this service + if $enable { + + # install required packages + if $packages { + ensure_packages($packages, {ensure => 'installed'}) + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage directories + file { [ $data_path ]: + ensure => directory, + owner => $user, + group => $group, + } + + # manage environment options file + file { $vars_file: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('vmcluster/options.erb'), + } + + # manage the systemd service + if $manage_service { + + # manage the subscribed resources + if 'tls' in $options and $options['tls'] == 'true' { + if 'tlsCertFile' in $options and 'tlsKeyFile' in $options { + # tls option AND certs listed, subscribe to the options file and certs + $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]] + }else{ + # tls option but no certs listed, just subscribe to the options file + warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vmselect options.') + $subscribe = [File[$vars_file]] + } + }else{ + # no tls option, just subscribe to the options file + $subscribe = [File[$vars_file]] + } + + # Use these in notifications or file resources + systemd::unit_file { 'vmselect.service': + content => template('vmcluster/vmselect.service.erb'), + enable => true, + active => true, + subscribe => $subscribe, + } + } + } +} diff --git a/modules/vmcluster/manifests/vmstorage.pp b/modules/vmcluster/manifests/vmstorage.pp new file mode 100644 index 0000000..b6e917d --- /dev/null +++ b/modules/vmcluster/manifests/vmstorage.pp @@ -0,0 +1,79 @@ +class vmcluster::vmstorage ( + Boolean $enable = false, + String $user = 'vmstorage', + String $group = 'vmstorage', + Boolean $manage_user = true, + Boolean $manage_service = true, + Array[String] $packages = ['vmstorage', 'vmutils'], + Stdlib::Absolutepath $exec_path = '/usr/bin/vmstorage', + Stdlib::Absolutepath $data_path = '/var/lib/vmstorage', + Stdlib::Absolutepath $vars_file = '/etc/default/vmstorage', + Hash[String, Variant[String, Array[String]]] $options = {}, +) { + + # if enabled, manage this service + if $enable { + + # install required packages + if $packages { + ensure_packages($packages, {ensure => 'installed'}) + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage directories + file { [ $data_path ]: + ensure => directory, + owner => $user, + group => $group, + } + + # manage environment options file + file { $vars_file: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('vmcluster/options.erb'), + } + + # manage the systemd service + if $manage_service { + + # manage the subscribed resources + if 'tls' in $options and $options['tls'] == 'true' { + if 'tlsCertFile' in $options and 'tlsKeyFile' in $options { + # tls option AND certs listed, subscribe to the options file and certs + $subscribe = [File[$vars_file],File[$options['tlsCertFile']], File[$options['tlsKeyFile']]] + }else{ + # tls option but no certs listed, just subscribe to the options file + warning('TLS is enabled but tlsCertFile or tlsKeyFile is missing from vmstorage options.') + $subscribe = [File[$vars_file]] + } + }else{ + # no tls option, just subscribe to the options file + $subscribe = [File[$vars_file]] + } + + # Use these in notifications or file resources + systemd::unit_file { 'vmstorage.service': + content => template('vmcluster/vmstorage.service.erb'), + enable => true, + active => true, + subscribe => $subscribe, + } + } + } +} diff --git a/modules/vmcluster/templates/options.erb b/modules/vmcluster/templates/options.erb new file mode 100644 index 0000000..f75377b --- /dev/null +++ b/modules/vmcluster/templates/options.erb @@ -0,0 +1,18 @@ +<% + opts = @options || {} + lines = [] + + opts.each do |key, val| + flag = "-#{key}" + + if val.is_a?(Array) + val.each do |v| + lines << "#{flag}=#{v}" + end + else + lines << "#{flag}=#{val}" + end + end +%> +OPTIONS="<%= lines.map { |l| l.lstrip }.join(" \\\n ") %>" + diff --git a/modules/vmcluster/templates/vmagent.scrape.yaml.erb b/modules/vmcluster/templates/vmagent.scrape.yaml.erb new file mode 100644 index 0000000..4bbf8c1 --- /dev/null +++ b/modules/vmcluster/templates/vmagent.scrape.yaml.erb @@ -0,0 +1,37 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + +scrape_configs: + - job_name: 'consul' + consul_sd_configs: + - server: 'consul.service.consul:8500' + token: <%= @consul_node_token %> + scheme: http + + relabel_configs: + # Only keep services with the `metrics` tag + - source_labels: [__meta_consul_tagpresent_metrics] + regex: true + action: keep + + # Use the service address and port + - source_labels: [__meta_consul_node, __meta_consul_service_port] + separator: ':' + target_label: __address__ + replacement: '${1}:${2}' + action: replace + + # Set scheme to metrics_scheme + - source_labels: [__meta_consul_tag_metrics_scheme] + target_label: __scheme__ + action: replace + + # Set path to /metrics + - target_label: __metrics_path__ + replacement: /metrics + + # Use metrics_job= as job label + - source_labels: [__meta_consul_tag_metrics_job] + target_label: job + action: replace diff --git a/modules/vmcluster/templates/vmagent.service.erb b/modules/vmcluster/templates/vmagent.service.erb new file mode 100644 index 0000000..a83a1be --- /dev/null +++ b/modules/vmcluster/templates/vmagent.service.erb @@ -0,0 +1,18 @@ +[Unit] +Description=VictoriaMetrics vmagent service +After=network.target + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +Restart=always +EnvironmentFile=<%= @vars_file %> +ExecStart=<%= @exec_path %> -remoteWrite.tmpDataPath=<%= @data_path %> $OPTIONS + +PrivateTmp=yes +NoNewPrivileges=yes +ProtectSystem=full + +[Install] +WantedBy=multi-user.target diff --git a/modules/vmcluster/templates/vminsert.service.erb b/modules/vmcluster/templates/vminsert.service.erb new file mode 100644 index 0000000..f619c9f --- /dev/null +++ b/modules/vmcluster/templates/vminsert.service.erb @@ -0,0 +1,18 @@ +[Unit] +Description=VictoriaMetrics vminsert service +After=network.target + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +Restart=always +EnvironmentFile=<%= @vars_file %> +ExecStart=<%= @exec_path %> $OPTIONS + +PrivateTmp=yes +NoNewPrivileges=yes +ProtectSystem=full + +[Install] +WantedBy=multi-user.target diff --git a/modules/vmcluster/templates/vmselect.service.erb b/modules/vmcluster/templates/vmselect.service.erb new file mode 100644 index 0000000..03e1507 --- /dev/null +++ b/modules/vmcluster/templates/vmselect.service.erb @@ -0,0 +1,18 @@ +[Unit] +Description=VictoriaMetrics vmselect service +After=network.target + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +Restart=always +EnvironmentFile=<%= @vars_file %> +ExecStart=<%= @exec_path %> -cacheDataPath=<%= @data_path %> $OPTIONS + +PrivateTmp=yes +NoNewPrivileges=yes +ProtectSystem=full + +[Install] +WantedBy=multi-user.target diff --git a/modules/vmcluster/templates/vmstorage.service.erb b/modules/vmcluster/templates/vmstorage.service.erb new file mode 100644 index 0000000..c349c83 --- /dev/null +++ b/modules/vmcluster/templates/vmstorage.service.erb @@ -0,0 +1,18 @@ +[Unit] +Description=VictoriaMetrics vmstorage service +After=network.target + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +Restart=always +EnvironmentFile=<%= @vars_file %> +ExecStart=<%= @exec_path %> -storageDataPath=<%= @data_path %> $OPTIONS + +PrivateTmp=yes +NoNewPrivileges=yes +ProtectSystem=full + +[Install] +WantedBy=multi-user.target