feat: create exporters module

- upgrade node_exporter, bring managed under exporters module
- upgrade postgres_exporter, bring managed under exporters module
- add flag to cleanup previous iterations of exporters from prometheus module
- fix issues with vmclusster: replication + dedup

hieradata/common.yaml

@@ -165,6 +165,7 @@ hiera_include:
   - profiles::accounts::rundeck
   - limits
   - sysctl::base
+  - exporters::node_exporter
 
 profiles::ntp::client::peers:
   - 0.au.pool.ntp.org
@@ -205,6 +206,9 @@ profiles::consul::client::node_rules:
   - resource: node
     segment: ''
     disposition: read
+  - resource: service
+    segment: node_exporter
+    disposition: write
 
 profiles::packages::include:
   bash-completion: {}
@@ -288,7 +292,8 @@ profiles::puppet::client::dns_alt_names:
 puppetdbapi: puppetdbapi.query.consul
 puppetdbsql: puppetdbsql.service.au-syd1.consul
 
-prometheus::node_exporter::export_scrape_job: true
+exporters::node_exporter::enable: true
+exporters::node_exporter::cleanup_old_node_exporter: true
 prometheus::systemd_exporter::export_scrape_job: true
 
 ssh::server::storeconfigs_enabled: false

hieradata/roles/infra/metrics/vminsert.yaml

@@ -4,7 +4,7 @@ hiera_include:
 
 vmcluster::vminsert::enable: true
 vmcluster::vminsert::options:
-  replicationFactor: '2'
+  replicationFactor: '5'
   httpListenAddr: ':8480'
   tls: 'true'
   tlsCertFile: '/etc/pki/tls/vault/certificate.crt'

hieradata/roles/infra/metrics/vmselect.yaml

@@ -7,8 +7,9 @@ vmcluster::vmselect::data_path: /data/vmselect-cache
 vmcluster::vmselect::options:
   search.skipSlowReplicas: 'true'
   search.denyPartialResponse: 'false'
-  replicationFactor: '2'
-  globalReplicationFactor: '2'
+  replicationFactor: '5'
+  globalReplicationFactor: '3'
+  dedup.minScrapeInterval: 15s
   tls: 'true'
   tlsCertFile: '/etc/pki/tls/vault/certificate.crt'
   tlsKeyFile: '/etc/pki/tls/vault/private.key'

hieradata/roles/infra/metrics/vmstorage.yaml

@@ -12,6 +12,7 @@ vmcluster::vmstorage::options:
   tlsMinVersion: 'TLS12'
   memory.allowedBytes: '1GiB'
   metrics.exposeMetadata: 'true'
+  dedup.minScrapeInterval: 15s
   envflag.enable: 'true'
 
 # additional altnames

hieradata/roles/infra/puppetdb/sql.yaml

@@ -56,6 +56,9 @@ profiles::consul::client::node_rules:
   - resource: session_prefix
     segment: ""
     disposition: write
+  - resource: service
+    segment: postgres_exporter
+    disposition: write
 
 profiles::yum::global::repos:
   postgresql-17:

hieradata/roles/infra/sql/patroni.yaml

@@ -26,3 +26,6 @@ profiles::consul::client::node_rules:
   - resource: session_prefix
     segment: ""
     disposition: write
+  - resource: service
+    segment: postgres_exporter
+    disposition: write

hieradata/roles/infra/sql/shared.yaml

@@ -41,6 +41,9 @@ profiles::consul::client::node_rules:
   - resource: session_prefix
     segment: ""
     disposition: write
+  - resource: service
+    segment: postgres_exporter
+    disposition: write
 
 profiles::yum::global::repos:
   postgresql-17:

modules/exporters/manifests/node_exporter.pp

@@ -0,0 +1,81 @@
+class exporters::node_exporter (
+  Boolean $enable         = false,
+  String  $user           = 'node_exporter',
+  String  $group          = 'node_exporter',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Stdlib::Port $port      = 9100,
+  Stdlib::Absolutepath $exec_path     = '/usr/bin/node_exporter',
+  Boolean $cleanup_old_node_exporter  = false,
+){
+
+  if $cleanup_old_node_exporter {
+    # remove the symlink
+    file {'/usr/local/bin/node_exporter':
+      ensure => absent
+    }
+    # remove the /opt/node_exporter-1.8.1.linux-amd64 directory
+    file {'/opt/node_exporter-1.8.1.linux-amd64':
+      ensure  => absent,
+      recurse => true,
+      force   => true,
+    }
+  }
+
+  if $enable {
+
+    # install required package
+    package {'node_exporter':
+      ensure => installed,
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'node_exporter.service':
+        content   => template('exporters/node_exporter.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => Package['node_exporter'],
+      }
+    }
+
+    # manage consul service
+    consul::service { 'node_exporter':
+      service_name => 'node_exporter',
+      address      => $facts['networking']['ip'],
+      port         => $port,
+      tags         => [
+        'metrics',
+        'metrics_scheme=http',
+        'metrics_job=node',
+      ],
+      checks       => [
+        {
+          id              => 'node_exporter_http_check',
+          name            => 'node_exporter HTTP Check',
+          http            => "http://${facts['networking']['fqdn']}:${port}",
+          method          => 'GET',
+          tls_skip_verify => true,
+          interval        => '10s',
+          timeout         => '1s',
+        },
+      ],
+    }
+  }
+}

modules/exporters/manifests/postgres_exporter.pp

@@ -0,0 +1,95 @@
+class exporters::postgres_exporter (
+  String        $db_pass,
+  String        $db_user  = 'postgres_exporter',
+  Stdlib::Host  $db_host  = $facts['networking']['ip'],
+  Stdlib::Port  $db_port  = 5432,
+  Boolean $enable         = false,
+  String  $user           = 'postgres_exporter',
+  String  $group          = 'postgres_exporter',
+  Boolean $manage_user    = true,
+  Boolean $manage_service = true,
+  Stdlib::Port $port      = 9187,
+  Stdlib::Absolutepath $exec_path     = '/usr/bin/postgres_exporter',
+  Stdlib::Absolutepath $vars_path     = '/etc/sysconfig/postgres_exporter',
+  Boolean $cleanup_old_postgres_exporter  = false,
+){
+
+  if $cleanup_old_postgres_exporter {
+    # remove the symlink
+    file {'/usr/local/bin/postgres_exporter':
+      ensure => absent
+    }
+    # remove the /opt/postgres_exporter-0.5.1.linux-amd64 directory
+    file {'/opt/postgres_exporter-0.5.1.linux-amd64':
+      ensure  => absent,
+      recurse => true,
+      force   => true,
+    }
+  }
+
+  if $enable {
+
+    # install required package
+    package {'postgres_exporter':
+      ensure => installed,
+    }
+
+    # manage the user/group
+    if $manage_user {
+      group { $group:
+        ensure => present,
+      }
+
+      user { $user:
+        ensure     => present,
+        shell      => '/usr/sbin/nologin',
+        groups     => $group,
+        managehome => true,
+      }
+    }
+
+    # manage the environment file
+    file { $vars_path:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('exporters/postgres_exporter_sysconfig.erb')
+    }
+
+    # manage the systemd service
+    if $manage_service {
+
+      # Use these in notifications or file resources
+      systemd::unit_file { 'postgres_exporter.service':
+        content   => template('exporters/postgres_exporter.service.erb'),
+        enable    => true,
+        active    => true,
+        subscribe => Package['postgres_exporter'],
+      }
+    }
+
+    # manage consul service
+    consul::service { 'postgres_exporter':
+      service_name => 'postgres_exporter',
+      address      => $facts['networking']['ip'],
+      port         => $port,
+      tags         => [
+        'metrics',
+        'metrics_scheme=http',
+        'metrics_job=postgres',
+      ],
+      checks       => [
+        {
+          id              => 'postgres_exporter_http_check',
+          name            => 'postgres_exporter HTTP Check',
+          http            => "http://${facts['networking']['fqdn']}:${port}",
+          method          => 'GET',
+          tls_skip_verify => true,
+          interval        => '10s',
+          timeout         => '1s',
+        },
+      ],
+    }
+  }
+}

modules/exporters/templates/node_exporter.service.erb

@@ -0,0 +1,15 @@
+[Unit]
+Description=Prometheus node_exporter
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=<%= @user %>
+Group=<%= @group %>
+ExecStart=<%= @exec_path %>
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

modules/exporters/templates/postgres_exporter.service.erb

@@ -0,0 +1,16 @@
+[Unit]
+Description=Prometheus postgres_exporter
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=<%= @user %>
+Group=<%= @group %>
+EnvironmentFile=<%= @vars_file %>
+ExecStart=<%= @exec_path %>
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

modules/exporters/templates/postgres_exporter_sysconfig.erb

@@ -0,0 +1,4 @@
+# THIS FILE IS MANAGED BY PUPPET
+DATA_SOURCE_URI="<%= @db_host %>:<%= @db_port %>/postgres?sslmode=disable"
+DATA_SOURCE_USER="<%= @db_user %>"
+DATA_SOURCE_PASS="<%= @db_pass %>"

site/profiles/manifests/base.pp

@@ -37,7 +37,6 @@ class profiles::base (
     include profiles::ssh::knownhosts
     include profiles::ssh::service
     include profiles::cloudinit::init
-    include profiles::metrics::default
     include profiles::helpers::node_lookup
     include profiles::consul::client
 

site/profiles/manifests/metrics/default.pp

@@ -1,11 +0,0 @@
-# profiles::metrics::default
-#
-# these exporters will be setup on all nodes
-class profiles::metrics::default (
-  Boolean $node_exporter      = true,
-  Boolean $systemd_exporter   = false,
-) {
-
-  if $node_exporter { include prometheus::node_exporter }
-  if $systemd_exporter { include prometheus::systemd_exporter }
-}

site/profiles/manifests/sql/patroni.pp

@@ -84,11 +84,11 @@ class profiles::sql::patroni (
   }
 
   if $postgres_exporter_enabled {
-    class { 'prometheus::postgres_exporter':
-      postgres_user     => $postgres_exporter_user,
-      postgres_pass     => $postgres_exporter_pass,
-      data_source_uri   => "${facts['networking']['ip']}:5432/postgres?sslmode=disable",
-      export_scrape_job => true,
+    class { 'exporters::postgres_exporter':
+      db_user                       => $postgres_exporter_user,
+      db_pass                       => $postgres_exporter_pass,
+      cleanup_old_postgres_exporter => true,
+      enable                        => true,
     }
   }
 }