diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 889dd4b..e110d0a 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -165,6 +165,7 @@ hiera_include: - profiles::accounts::rundeck - limits - sysctl::base + - exporters::node_exporter profiles::ntp::client::peers: - 0.au.pool.ntp.org @@ -205,6 +206,9 @@ profiles::consul::client::node_rules: - resource: node segment: '' disposition: read + - resource: service + segment: node_exporter + disposition: write profiles::packages::include: bash-completion: {} @@ -288,7 +292,8 @@ profiles::puppet::client::dns_alt_names: puppetdbapi: puppetdbapi.query.consul puppetdbsql: puppetdbsql.service.au-syd1.consul -prometheus::node_exporter::export_scrape_job: true +exporters::node_exporter::enable: true +exporters::node_exporter::cleanup_old_node_exporter: true prometheus::systemd_exporter::export_scrape_job: true ssh::server::storeconfigs_enabled: false diff --git a/hieradata/roles/infra/metrics/vminsert.yaml b/hieradata/roles/infra/metrics/vminsert.yaml index ac5869e..2b26bf0 100644 --- a/hieradata/roles/infra/metrics/vminsert.yaml +++ b/hieradata/roles/infra/metrics/vminsert.yaml @@ -4,7 +4,7 @@ hiera_include: vmcluster::vminsert::enable: true vmcluster::vminsert::options: - replicationFactor: '2' + replicationFactor: '5' httpListenAddr: ':8480' tls: 'true' tlsCertFile: '/etc/pki/tls/vault/certificate.crt' diff --git a/hieradata/roles/infra/metrics/vmselect.yaml b/hieradata/roles/infra/metrics/vmselect.yaml index aea211f..7f3a1ec 100644 --- a/hieradata/roles/infra/metrics/vmselect.yaml +++ b/hieradata/roles/infra/metrics/vmselect.yaml @@ -7,8 +7,9 @@ vmcluster::vmselect::data_path: /data/vmselect-cache vmcluster::vmselect::options: search.skipSlowReplicas: 'true' search.denyPartialResponse: 'false' - replicationFactor: '2' - globalReplicationFactor: '2' + replicationFactor: '5' + globalReplicationFactor: '3' + dedup.minScrapeInterval: 15s tls: 'true' tlsCertFile: '/etc/pki/tls/vault/certificate.crt' tlsKeyFile: '/etc/pki/tls/vault/private.key' diff --git a/hieradata/roles/infra/metrics/vmstorage.yaml b/hieradata/roles/infra/metrics/vmstorage.yaml index ca84896..5082873 100644 --- a/hieradata/roles/infra/metrics/vmstorage.yaml +++ b/hieradata/roles/infra/metrics/vmstorage.yaml @@ -12,6 +12,7 @@ vmcluster::vmstorage::options: tlsMinVersion: 'TLS12' memory.allowedBytes: '1GiB' metrics.exposeMetadata: 'true' + dedup.minScrapeInterval: 15s envflag.enable: 'true' # additional altnames diff --git a/hieradata/roles/infra/puppetdb/sql.yaml b/hieradata/roles/infra/puppetdb/sql.yaml index 85c69a7..1c9719d 100644 --- a/hieradata/roles/infra/puppetdb/sql.yaml +++ b/hieradata/roles/infra/puppetdb/sql.yaml @@ -56,6 +56,9 @@ profiles::consul::client::node_rules: - resource: session_prefix segment: "" disposition: write + - resource: service + segment: postgres_exporter + disposition: write profiles::yum::global::repos: postgresql-17: diff --git a/hieradata/roles/infra/sql/patroni.yaml b/hieradata/roles/infra/sql/patroni.yaml index f962a62..6f83721 100644 --- a/hieradata/roles/infra/sql/patroni.yaml +++ b/hieradata/roles/infra/sql/patroni.yaml @@ -26,3 +26,6 @@ profiles::consul::client::node_rules: - resource: session_prefix segment: "" disposition: write + - resource: service + segment: postgres_exporter + disposition: write diff --git a/hieradata/roles/infra/sql/shared.yaml b/hieradata/roles/infra/sql/shared.yaml index 7a124be..88cf1d5 100644 --- a/hieradata/roles/infra/sql/shared.yaml +++ b/hieradata/roles/infra/sql/shared.yaml @@ -41,6 +41,9 @@ profiles::consul::client::node_rules: - resource: session_prefix segment: "" disposition: write + - resource: service + segment: postgres_exporter + disposition: write profiles::yum::global::repos: postgresql-17: diff --git a/modules/exporters/manifests/node_exporter.pp b/modules/exporters/manifests/node_exporter.pp new file mode 100644 index 0000000..41f361e --- /dev/null +++ b/modules/exporters/manifests/node_exporter.pp @@ -0,0 +1,81 @@ +class exporters::node_exporter ( + Boolean $enable = false, + String $user = 'node_exporter', + String $group = 'node_exporter', + Boolean $manage_user = true, + Boolean $manage_service = true, + Stdlib::Port $port = 9100, + Stdlib::Absolutepath $exec_path = '/usr/bin/node_exporter', + Boolean $cleanup_old_node_exporter = false, +){ + + if $cleanup_old_node_exporter { + # remove the symlink + file {'/usr/local/bin/node_exporter': + ensure => absent + } + # remove the /opt/node_exporter-1.8.1.linux-amd64 directory + file {'/opt/node_exporter-1.8.1.linux-amd64': + ensure => absent, + recurse => true, + force => true, + } + } + + if $enable { + + # install required package + package {'node_exporter': + ensure => installed, + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage the systemd service + if $manage_service { + + # Use these in notifications or file resources + systemd::unit_file { 'node_exporter.service': + content => template('exporters/node_exporter.service.erb'), + enable => true, + active => true, + subscribe => Package['node_exporter'], + } + } + + # manage consul service + consul::service { 'node_exporter': + service_name => 'node_exporter', + address => $facts['networking']['ip'], + port => $port, + tags => [ + 'metrics', + 'metrics_scheme=http', + 'metrics_job=node', + ], + checks => [ + { + id => 'node_exporter_http_check', + name => 'node_exporter HTTP Check', + http => "http://${facts['networking']['fqdn']}:${port}", + method => 'GET', + tls_skip_verify => true, + interval => '10s', + timeout => '1s', + }, + ], + } + } +} diff --git a/modules/exporters/manifests/postgres_exporter.pp b/modules/exporters/manifests/postgres_exporter.pp new file mode 100644 index 0000000..5fb4425 --- /dev/null +++ b/modules/exporters/manifests/postgres_exporter.pp @@ -0,0 +1,95 @@ +class exporters::postgres_exporter ( + String $db_pass, + String $db_user = 'postgres_exporter', + Stdlib::Host $db_host = $facts['networking']['ip'], + Stdlib::Port $db_port = 5432, + Boolean $enable = false, + String $user = 'postgres_exporter', + String $group = 'postgres_exporter', + Boolean $manage_user = true, + Boolean $manage_service = true, + Stdlib::Port $port = 9187, + Stdlib::Absolutepath $exec_path = '/usr/bin/postgres_exporter', + Stdlib::Absolutepath $vars_path = '/etc/sysconfig/postgres_exporter', + Boolean $cleanup_old_postgres_exporter = false, +){ + + if $cleanup_old_postgres_exporter { + # remove the symlink + file {'/usr/local/bin/postgres_exporter': + ensure => absent + } + # remove the /opt/postgres_exporter-0.5.1.linux-amd64 directory + file {'/opt/postgres_exporter-0.5.1.linux-amd64': + ensure => absent, + recurse => true, + force => true, + } + } + + if $enable { + + # install required package + package {'postgres_exporter': + ensure => installed, + } + + # manage the user/group + if $manage_user { + group { $group: + ensure => present, + } + + user { $user: + ensure => present, + shell => '/usr/sbin/nologin', + groups => $group, + managehome => true, + } + } + + # manage the environment file + file { $vars_path: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('exporters/postgres_exporter_sysconfig.erb') + } + + # manage the systemd service + if $manage_service { + + # Use these in notifications or file resources + systemd::unit_file { 'postgres_exporter.service': + content => template('exporters/postgres_exporter.service.erb'), + enable => true, + active => true, + subscribe => Package['postgres_exporter'], + } + } + + # manage consul service + consul::service { 'postgres_exporter': + service_name => 'postgres_exporter', + address => $facts['networking']['ip'], + port => $port, + tags => [ + 'metrics', + 'metrics_scheme=http', + 'metrics_job=postgres', + ], + checks => [ + { + id => 'postgres_exporter_http_check', + name => 'postgres_exporter HTTP Check', + http => "http://${facts['networking']['fqdn']}:${port}", + method => 'GET', + tls_skip_verify => true, + interval => '10s', + timeout => '1s', + }, + ], + } + } +} diff --git a/modules/exporters/templates/node_exporter.service.erb b/modules/exporters/templates/node_exporter.service.erb new file mode 100644 index 0000000..68b6cd2 --- /dev/null +++ b/modules/exporters/templates/node_exporter.service.erb @@ -0,0 +1,15 @@ +[Unit] +Description=Prometheus node_exporter +Wants=network-online.target +After=network-online.target + +[Service] +User=<%= @user %> +Group=<%= @group %> +ExecStart=<%= @exec_path %> +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/modules/exporters/templates/postgres_exporter.service.erb b/modules/exporters/templates/postgres_exporter.service.erb new file mode 100644 index 0000000..33cd8b2 --- /dev/null +++ b/modules/exporters/templates/postgres_exporter.service.erb @@ -0,0 +1,16 @@ +[Unit] +Description=Prometheus postgres_exporter +Wants=network-online.target +After=network-online.target + +[Service] +User=<%= @user %> +Group=<%= @group %> +EnvironmentFile=<%= @vars_file %> +ExecStart=<%= @exec_path %> +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/modules/exporters/templates/postgres_exporter_sysconfig.erb b/modules/exporters/templates/postgres_exporter_sysconfig.erb new file mode 100644 index 0000000..48de0e1 --- /dev/null +++ b/modules/exporters/templates/postgres_exporter_sysconfig.erb @@ -0,0 +1,4 @@ +# THIS FILE IS MANAGED BY PUPPET +DATA_SOURCE_URI="<%= @db_host %>:<%= @db_port %>/postgres?sslmode=disable" +DATA_SOURCE_USER="<%= @db_user %>" +DATA_SOURCE_PASS="<%= @db_pass %>" diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp index 890fa6f..4c5c70f 100644 --- a/site/profiles/manifests/base.pp +++ b/site/profiles/manifests/base.pp @@ -37,7 +37,6 @@ class profiles::base ( include profiles::ssh::knownhosts include profiles::ssh::service include profiles::cloudinit::init - include profiles::metrics::default include profiles::helpers::node_lookup include profiles::consul::client diff --git a/site/profiles/manifests/metrics/default.pp b/site/profiles/manifests/metrics/default.pp deleted file mode 100644 index f6def27..0000000 --- a/site/profiles/manifests/metrics/default.pp +++ /dev/null @@ -1,11 +0,0 @@ -# profiles::metrics::default -# -# these exporters will be setup on all nodes -class profiles::metrics::default ( - Boolean $node_exporter = true, - Boolean $systemd_exporter = false, -) { - - if $node_exporter { include prometheus::node_exporter } - if $systemd_exporter { include prometheus::systemd_exporter } -} diff --git a/site/profiles/manifests/sql/patroni.pp b/site/profiles/manifests/sql/patroni.pp index 079bd3d..e8d9939 100644 --- a/site/profiles/manifests/sql/patroni.pp +++ b/site/profiles/manifests/sql/patroni.pp @@ -84,11 +84,11 @@ class profiles::sql::patroni ( } if $postgres_exporter_enabled { - class { 'prometheus::postgres_exporter': - postgres_user => $postgres_exporter_user, - postgres_pass => $postgres_exporter_pass, - data_source_uri => "${facts['networking']['ip']}:5432/postgres?sslmode=disable", - export_scrape_job => true, + class { 'exporters::postgres_exporter': + db_user => $postgres_exporter_user, + db_pass => $postgres_exporter_pass, + cleanup_old_postgres_exporter => true, + enable => true, } } }