diff --git a/Puppetfile b/Puppetfile index 9677851..d57f3c2 100644 --- a/Puppetfile +++ b/Puppetfile @@ -53,6 +53,7 @@ mod 'broadinstitute-certs', '3.0.1' mod 'stm-file_capability', '6.0.0' mod 'h0tw1r3-gitea', '3.2.0' mod 'rehan-mkdir', '2.0.0' +mod 'tailoredautomation-patroni', '2.0.0' mod 'bind', :git => 'https://git.service.au-syd1.consul/unkinben/puppet-bind.git', diff --git a/hieradata/common.yaml b/hieradata/common.yaml index b7b1894..518df23 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -180,7 +180,6 @@ profiles::packages::include: curl: {} dstat: {} expect: {} - gcc: {} gzip: {} git: {} htop: {} diff --git a/hieradata/roles/infra/metrics/prometheus.yaml b/hieradata/roles/infra/metrics/prometheus.yaml index 5e29220..bce99c2 100644 --- a/hieradata/roles/infra/metrics/prometheus.yaml +++ b/hieradata/roles/infra/metrics/prometheus.yaml @@ -9,4 +9,5 @@ profiles::metrics::server::scrape_jobs: - puppetdb - systemd - haproxy + - postgres profiles::metrics::server::localstorage: /data/prometheus diff --git a/hieradata/roles/infra/sql/patroni.eyaml b/hieradata/roles/infra/sql/patroni.eyaml new file mode 100644 index 0000000..2602f02 --- /dev/null +++ b/hieradata/roles/infra/sql/patroni.eyaml @@ -0,0 +1,4 @@ +--- +profiles::sql::patroni::superuser_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAImaYEH6ktU6KYDu96OtuJu11FbmE+b9YwrkJiv72QfuVivR7lGfieRKcvkrq3IsNcwhhnl1lzyMZHteib7jLEIiaq59AZFiZyfF2E5bhNYfW4QcDJd4QOheU2awkZkl6oaoMUxgWOnqvihYVLDfoJ0lj6hBTFuqIzO/KU+AJ4NMO3+/+AK9+HB0u/8Iyuev4RQBkvaRAoszCcFCWTAZJTmhOgWe4xJIxfbh0/5k/dmT5WQ1JPEQK7hnVly9iToROZJDjuyndNHbrhCQUg4+DYMPS2fZVWvcESfxN8lJjBFPojj9ZbG5mLlq1e4A4KiZNwHfA1V4D88VWOkRg65XtZDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBzToImdK5/pk3UeZPyavjTgDCcYBzwY/g353Pbh0xr/we8KmvsQEtfxPDuPm4Kv4hsD5X0dHu2nGzBAZq5uWcE3RE=] +profiles::sql::patroni::replication_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWyCj+7WfzpTcpBg6uQ5ykGmLZmb/avW3Pc+VWj9bGvxSQCA8LA6HJlEhhL3mrJSTGUyHLgeEebEup9AVHe2k2l/JHIvhyfx7LI+mNDp8u5p40pM6ZxTdIJFOZmOS/nGjAR6mTv6Ennhpw4sWSDYXU0mJPTHGAked2FXV1xsS0zpTY7hccJHuww5ixOw6jP8E1Pu0ex4LmefOXApowf0jZ2pARndlsXwZldahUHIF48XejclpgCK9rTrb4eQsOZr5ozcj0BBpWg/JKNkQt8mQU5l5/z0GDT08Op8g6MVdJuOWr92uPqjc8sydrz0QAx4l8t1KY2fMWK7BPKqSdcOxiDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDEZBNd56BHVGRVfHDPPwZHgDAZKnqicbF/MVKPi1PwwyHrXMW/fWqocgr1zWx6RXWgXICqjJdEFXwFerXXb39RSDg=] +profiles::sql::patroni::postgres_exporter_pass: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAL5brQt9CGFU7okDXZWF1jL1j+RbrQZhKfzGWyWl+SqRK6q+xH0LIzYQhOAji7tlDBzvFZpzglmzj0xDrAkQA46jg1DkR5+9Ozru9jL1nhg/6z/F54DlhAG7Ui0hjgSLal79VABLXa/cb9xJThx97b9xoOW+/vpfSKa4izFtkN9fliClFTVafxLlLLD/yABW99aq1OK+9MyCsppvs/rjWbXvjEKL+C0jawh4dBnc+tYJMHC/k5NIK0th4A/zSYVH5q6gFakpxrV2ubETIbVTDncC8zfRLhnrikZYNbCy5PuJb2a4vW1O0AOzUWqvqbRkWpYF7dJB9fzW/Tu8f8d10KTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAJS5PcA9aJvjGIfKSciLLpgDAU7xXGF+Sj+g1ABMvsenEmgXsdSKVU9ZYusIiGnPdFdN4EF9usi7g4SahochlG0NU=] diff --git a/hieradata/roles/infra/sql/patroni.yaml b/hieradata/roles/infra/sql/patroni.yaml new file mode 100644 index 0000000..f925067 --- /dev/null +++ b/hieradata/roles/infra/sql/patroni.yaml @@ -0,0 +1,28 @@ +--- +profiles::yum::global::repos: + postgresql-15: + name: postgresql-15 + descr: postgresql-15 repository + target: /etc/yum.repos.d/postgresql.repo + baseurl: https://edgecache.query.consul/postgres/yum/15/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture} + gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL + postgresql-common: + name: postgresql-common + descr: postgresql-common repository + target: /etc/yum.repos.d/postgresql.repo + baseurl: https://edgecache.query.consul/postgres/yum/common/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture} + gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL + +profiles::sql::patroni::cluster_name: "patroni-%{facts.environment}" +profiles::sql::patroni::postgres_exporter_enabled: true +profiles::sql::patroni::postgres_exporter_user: postgres_exporter +profiles::consul::client::node_rules: + - resource: service_prefix + segment: "%{hiera('profiles::sql::patroni::cluster_name')}" + disposition: write + - resource: key_prefix + segment: "service/%{hiera('profiles::sql::patroni::cluster_name')}" + disposition: write + - resource: session_prefix + segment: "" + disposition: write diff --git a/site/profiles/manifests/sql/patroni.pp b/site/profiles/manifests/sql/patroni.pp new file mode 100644 index 0000000..69449cb --- /dev/null +++ b/site/profiles/manifests/sql/patroni.pp @@ -0,0 +1,95 @@ +# profiles::sql::patroni +class profiles::sql::patroni ( + String $cluster_name, + String $superuser_password, + String $replication_password, + String $superuser_username = 'postgres', + String $replication_username = 'repl', + String $pgsql_version = '15', + Stdlib::Absolutepath $pgsql_data_base = '/data/pgsql', + Stdlib::Absolutepath $pgsql_data_dir = "${pgsql_data_base}/${pgsql_version}/data", + Boolean $use_consul = true, + String $consul_host = 'localhost', + Stdlib::Port $consul_port = 8500, + Enum['http','https'] $consul_scheme = 'http', + Variant[Undef,String] $consul_token = undef, + Boolean $consul_verify = false, + Boolean $consul_register_service = true, + String $consul_service_check_interval = '5s', + String $consul_cacert = '/etc/pki/ca-trust/source/anchors/vaultcaroot.pem', + Boolean $postgres_exporter_enabled = false, + Optional[String] $postgres_exporter_user = undef, + Optional[String] $postgres_exporter_pass = undef, +){ + + # disable the postgresql dnf module for el8+ + if $facts['os']['family'] == 'RedHat' and $facts['os']['release']['major'] >= '8' { + # based on https://github.com/puppetlabs/puppetlabs-postgresql/blob/main/manifests/dnfmodule.pp + package { 'postgresql dnf module': + ensure => 'disabled', + name => 'postgresql', + provider => 'dnfmodule', + before => Class['patroni'], + } + } + + # prepare data path + mkdir::p {$pgsql_data_dir:} + file {$pgsql_data_dir: + ensure => 'directory', + owner => 'postgres', + group => 'postgres', + mode => '0700', + require => Class['patroni'], + } + + # manage patroni + class { 'patroni': + scope => $cluster_name, + use_consul => $use_consul, + consul_host => $consul_host, + consul_port => $consul_port, + consul_scheme => $consul_scheme, + consul_token => $consul_token, + consul_verify => $consul_verify, + consul_register_service => $consul_register_service, + consul_service_check_interval => $consul_service_check_interval, + consul_cacert => $consul_cacert, + manage_python => false, + pgsql_connect_address => "${facts['networking']['fqdn']}:5432", + restapi_connect_address => "${facts['networking']['fqdn']}:8008", + postgresql_version => $pgsql_version, + pgsql_data_dir => $pgsql_data_dir, + pgsql_pgpass_path => '/var/lib/pgsql/pgpass', + pgsql_parameters => { + 'max_connections' => 5000, + }, + bootstrap_pg_hba => [ + 'local all postgres ident', + 'host all all 0.0.0.0/0 md5', + 'host replication repl 0.0.0.0/0 md5', + ], + pgsql_pg_hba => [ + 'local all postgres ident', + 'host all all 0.0.0.0/0 md5', + 'host replication repl 0.0.0.0/0 md5', + ], + superuser_username => $superuser_username, + superuser_password => $superuser_password, + replication_username => $replication_username, + replication_password => $replication_password, + require => [ + Yumrepo["postgresql-${pgsql_version}"], + Yumrepo['postgresql-common'] + ], + } + + if $postgres_exporter_enabled { + class { 'prometheus::postgres_exporter': + postgres_user => $postgres_exporter_user, + postgres_pass => $postgres_exporter_pass, + data_source_uri => "${facts['networking']['ip']}:5432/postgres?sslmode=disable", + export_scrape_job => true, + } + } +} diff --git a/site/roles/manifests/infra/sql/patroni.pp b/site/roles/manifests/infra/sql/patroni.pp new file mode 100644 index 0000000..a3f83e3 --- /dev/null +++ b/site/roles/manifests/infra/sql/patroni.pp @@ -0,0 +1,12 @@ +# a role to deploy a postgresql/patroni node +class roles::infra::sql::patroni { + if $facts['firstrun'] { + include profiles::defaults + include profiles::firstrun::init + }else{ + include profiles::defaults + include profiles::base + include profiles::base::datavol + include profiles::sql::patroni + } +}