Merge pull request 'feat: setup galera cluster member profile' (#89) from neoloc/mariadbgalera into develop

Reviewed-on: unkinben/puppet-prod#89

Puppetfile

@@ -14,6 +14,7 @@ mod 'puppetlabs-postgresql', '9.1.0'
 mod 'puppetlabs-firewall', '6.0.0'
 mod 'puppetlabs-accounts', '8.1.0'
 mod 'puppetlabs-mysql', '15.0.0'
+mod 'puppetlabs-xinetd', '3.4.1'
 
 # puppet
 mod 'puppet-python', '7.0.0'

hieradata/common.yaml

@@ -42,6 +42,7 @@ profiles::packages::base::add:
   - pv
   - rsync
   - screen
+  - socat
   - strace
   - sysstat
   - tmux

hieradata/roles/infra/sql/galera.eyaml

@@ -0,0 +1,3 @@
+---
+profiles::sql::galera_member::root_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAdR+LgWom6UO3PhwmlkpttcJUkzvqsfJVGRYhNfbBtmzITSW2obN9q4ifCPweGA78+zsd8WzG8Ydd2vNsALLl0LogLHt2vpGnv3xomJROQYI03ipc4ih0mIcFii2m7f/quumbPzT4m7rotACtQtquWeYIXcTW6LMqAesLE97kTtSeTIcYmzOLAmkY0kWGMPsik3H6dJbjIFNtdQpu1e3Ah/2+dgbrr1zDzcPArk/5eb7RwMQpLGZHwvbXrNTCxpzAUA7ZclmKRT8SyDmOKAS/WTMe8Z6khp97cmT5N32HH3KHinSdxp8g40XrN4Ue86feEPxwKbeFF3Fzq4B7MYVSyDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBHcYpDTcYJHQX8iz3gvAbwgDA6h78+zGZ6Yn3VveL29fCzsKEvkalRGnHDcu8sZunNYbDrZDIrAdo/zI7Tivvb+TQ=]
+profiles::sql::galera_member::status_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAj0Zjb+XijI6Fqxv9nemJrONvuqEr2uG7dRtZ/ZSDIQCFDAAMilM6EKg+6hMdHVAno45qLDsmJUUgznEQEPe7qkvFKNzzZecGGLdyj0FVU9YMDL69qfcpTxhXxA7mxE+LrtbpArRNjWAgiiRv1REvo54ZdsMThLFrDvQG6myDRaNaWxQ9RWQ1o+oy57AYwrurNgsM8ziOEU3ZfF+ax1zGA+GlGgIiM6XW+w5aH4tLdaUbvfYhBZpGaa0Wh594TSDlzBslfmO4gx6076xUua0pi71ZeMfx63kedTyjj2k07C3SLKpknm+FqYG0ZhdEMSscCodnys/KYT3qiN5fStWN3zBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCCeXDpaOYDKhr1Q7A8I30ygDC7BBHB/UtDJLjuGZ4cll0MsfBlrQwAGDJm0j25JnsLCYqBvz1XjxFs1JhKJXLe42c=]

hieradata/roles/infra/sql/galera.yaml

@@ -0,0 +1,11 @@
+---
+profiles::sql::galera_member::cluster_name: galera01
+profiles::sql::galera_member::galera_master: prodinf01n29.main.unkin.net
+profiles::sql::galera_member::configure_firewall: false
+profiles::sql::galera_member::wsrep_sst_method: rsync
+profiles::sql::galera_member::galera_members_lookup: true
+profiles::sql::galera_member::galera_members_role: roles::infra::sql::galera
+profiles::sql::galera_member::datadir: /data/mariadb
+profiles::sql::galera_member::innodb_buffer_pool_size: 256M
+profiles::sql::galera_member::innodb_file_per_table: 1
+profiles::sql::galera_member::package_name: mariadb-galera-server

site/profiles/lib/facter/mariadb_active.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# create a boolean for when the mariadb service is active
+require 'English'
+
+Facter.add('mariadb_active') do
+  setcode do
+    system('systemctl is-active --quiet mariadb')
+    $CHILD_STATUS.exitstatus.zero?
+  end
+end

site/profiles/lib/facter/mariadb_datapath.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# check if the /etc/my.cnf.d/server.cnf file exists,
+# open it and search for the 'datadir =' option
+# store the datadir value as this fact
+require 'facter'
+
+Facter.add('mariadb_datapath') do
+  setcode do
+    if File.exist?('/etc/my.cnf.d/server.cnf')
+      datadir = nil
+      File.foreach('/etc/my.cnf.d/server.cnf') do |line|
+        match = line.match(/^\s*datadir\s*=\s*(.+)\s*$/)
+        if match
+          datadir = match[1].strip
+          break
+        end
+      end
+      datadir
+    end
+  end
+end

site/profiles/lib/facter/mariadb_galera_active.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# check if the mariadb server exists
+# check if the mariadb_datapath fact exists, else set /var/lib/mysql as the datapath
+# check if the galera grastate.dat file exists, identifying if galera is boostrapped
+require 'facter'
+
+if system('systemctl is-active --quiet mariadb')
+
+  Facter.add('mariadb_galera_active') do
+    setcode do
+      mariadb_datapath = Facter.value(:mariadb_datapath) || '/var/lib/mysql'
+      File.exist?("#{mariadb_datapath}/grastate.dat")
+    end
+  end
+end

site/profiles/lib/facter/mariadb_installed.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# create boolean for if mariadb is installed based of the default service file
+Facter.add('mariadb_installed') do
+  setcode do
+    File.exist?('/usr/lib/systemd/system/mariadb.service')
+  end
+end

site/profiles/manifests/selinux/mysqld.pp

@@ -0,0 +1,44 @@
+# profiles::selinux::mysqld
+# selinux settings for mysqld and galera
+class profiles::selinux::mysqld (
+  Stdlib::Absolutepath  $datadir = '/var/lib/mysql',
+  Boolean               $persistent = true,
+  Boolean               $mysql_connect_any = true,
+  Boolean               $selinuxuser_mysql_connect_enabled = true,
+  String                $selinux_mode = 'enforcing',
+){
+  # include packages that are required
+  include profiles::packages::selinux
+
+  # setenforce
+  class { 'profiles::selinux::setenforce':
+    mode => $selinux_mode,
+  }
+
+  # set mysqld_db_t to all files under the datadir
+  selinux::fcontext { $datadir:
+    ensure   => 'present',
+    seltype  => 'mysqld_db_t',
+    pathspec => "${datadir}(/.*)?",
+  }
+
+  # make sure we can connect to mysql on the local system
+  selboolean { 'selinuxuser_mysql_connect_enabled':
+    persistent => $persistent,
+    value      => $selinuxuser_mysql_connect_enabled,
+  }
+
+  # make sure mysql can connect to other hosts
+  selboolean { 'mysql_connect_any':
+    persistent => $persistent,
+    value      => $mysql_connect_any,
+  }
+
+  exec { "restorecon_${datadir}":
+    path        => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'],
+    command     => "restorecon -Rv ${datadir}",
+    refreshonly => true,
+    subscribe   => Selinux::Fcontext[$datadir],
+  }
+}
+

site/profiles/manifests/selinux/setenforce.pp

@@ -0,0 +1,9 @@
+# profiles::selinux::setenforce
+class profiles::selinux::setenforce (
+  Enum['enforcing', 'permissive', 'disabled'] $mode = 'enforcing',
+) {
+  class { 'selinux':
+    mode => $mode,
+  }
+}
+

site/profiles/manifests/sql/galera_member.pp

@@ -0,0 +1,215 @@
+# profiles::sql::galera_member
+class profiles::sql::galera_member (
+  String  $cluster_name,
+  String  $root_password,
+  String  $status_password,
+  Enum[
+    'mariabackup',
+    'mysqldump',
+    'rsync',
+    'skip',
+    'xtrabackup',
+    'xtrabackup-v2'
+  ]       $wsrep_sst_method         = 'xtrabackup-v2',
+  Integer $mysql_port               = 3306,
+  Boolean $galera_members_lookup    = false,
+  String  $galera_members_role      = undef,
+  String  $galera_master            = undef,
+  Array   $galera_servers           = [],
+  Boolean $configure_firewall       = false,
+  Integer $wsrep_state_transfer_port     = 4444,
+  Integer $wsrep_inc_state_transfer_port = 4568,
+  Integer $wsrep_group_comm_port         = 4567,
+  String  $innodb_buffer_pool_size  = '256M',
+  Integer $innodb_file_per_table    = 1,
+  Integer $innodb_autoinc_lock_mode = 2,
+  Stdlib::IP::Address $local_ip     = $facts['networking']['ip'],
+  Stdlib::Absolutepath $datadir     = '/var/lib/mysql',
+  Hash    $override_options_mysqld  = {},
+  Hash    $override_options_galera  = {},
+  Boolean $package_manage           = true,
+  String  $package_name             = 'mariadb-server',
+  Boolean $epel_needed              = false,
+  Boolean $selinux                  = true,
+  Boolean $manage_repo              = true,
+) {
+
+  # check that the master is named
+  unless !($galera_master == undef) {
+    fail("galera_master must be provided for ${title}")
+  }
+
+  # if lookup is enabled
+  if $galera_members_lookup {
+
+    # check that the role is also set
+    unless !($galera_members_role == undef) {
+      fail("galera_members_role must be provided for ${title} when galera_members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    $servers_array = sort(query_nodes("enc_role='${galera_members_role}'", 'networking.fqdn'))
+
+  # else use provided array from params
+  }else{
+    $servers_array = $galera_servers
+  }
+
+  # if its not an empty array. Give puppetdb a chance to be populated with data.
+  if length($servers_array) >= 3 {
+
+    # if selinux is defined, manage it
+    if $selinux {
+
+      # set permissive on first run, as we need mariadb installed/started at a custom path before adding fcontext
+      if $::facts['mariadb_acti'] { $selinux_mode = 'enforcing' }else{ $selinux_mode = 'permissive' }
+
+      # call the mysqld selinux class
+      class { 'profiles::selinux::mysqld':
+        datadir      => $datadir,
+        selinux_mode => $selinux_mode,
+        require      => Class['Mysql::Server'],
+      }
+    }
+
+    # check if this is the master_node
+    if $galera_master == $::facts['networking']['fqdn'] {
+      $galera_master_bool = true
+    }else{
+      $galera_master_bool = false
+    }
+
+    # find bootstrap status for servers
+    $bootstrap_array = puppetdb_query("inventory[certname, facts] { facts.enc_role = '${galera_members_role}' }").map |$node| {
+      {
+        'fqdn'      => $node['certname'],
+        'bootstrap' => $node['facts']['mariadb_galera_active'],
+      }
+    }
+
+    # determine if the cluster is bootstrapped
+    $cluster_bootstrapped = $bootstrap_array.any |$server| {
+      $server['fqdn'] == $galera_master and $server['bootstrap'] == true
+    }
+
+    # for setting puppetlabs-mysql params
+    # ['mysqld']['datadir'] = /var/lib/mysql
+    # TODO move to a params class later, mysql and galera
+    $default_override_options_mysqld = {
+      'mysqld' => {
+        'datadir'                  => $datadir,
+        'innodb_buffer_pool_size'  => $innodb_buffer_pool_size,
+        'innodb_file_per_table'    => $innodb_file_per_table,
+        'innodb_autoinc_lock_mode' => $innodb_autoinc_lock_mode,
+        'binlog_format'            => 'ROW',
+        'default-storage-engine'   => 'innodb',
+        'query_cache_size'         => '0',
+        'query_cache_type'         => '0'
+      }
+    }
+    $default_override_options_galera = {
+      'galera' => {
+        'wsrep_on'               => 'ON',
+        'wsrep_node_name'        => $::facts['networking']['hostname'],
+        'wsrep_provider'         => '/usr/lib64/galera/libgalera_smm.so',
+        'wsrep_cluster_name'     => $cluster_name,
+        'wsrep_cluster_address'  => "gcomm://${join($servers_array, ',')}",
+        'wsrep_sst_method'       => $wsrep_sst_method,
+        'wsrep_provider_options' => ["ist.recv_addr=${local_ip}:${wsrep_inc_state_transfer_port}", "ist.recv_bind=${local_ip}", ''],
+        'wsrep_node_address'     => "${local_ip}:${wsrep_group_comm_port}"
+      }
+    }
+
+    # merge the mysqld/galera defaults with the $override_options_{mysqld|galera}
+    $merged_overrides_mysqld_only = merge($default_override_options_mysqld, $override_options_mysqld)
+    $merged_overrides_galera_only = merge($default_override_options_mysqld, $override_options_mysqld)
+
+    # merge both galera and mariadb
+    $merged_overrides_both = merge($default_override_options_galera, $merged_overrides_mysqld_only)
+
+    # prepare non-master cluster members
+    if $::facts['mariadb_installed'] and ! $galera_master_bool {
+
+      # set service manage/enabled to match $cluster_bootstrapped
+      $real_service_manage = $cluster_bootstrapped
+      $real_service_enabled = $cluster_bootstrapped
+
+      # if cluster master is bootstrapped, add these nodes to the cluster
+      if $cluster_bootstrapped {
+        $merged_overrides = $merged_overrides_both
+      }else{
+        $merged_overrides = $merged_overrides_mysqld_only
+      }
+
+      # if cluster is boostrapped, but galera is not active on this node, then
+      # restart mariadb after mysql class reconfigures mariadb
+      if $cluster_bootstrapped and $::facts['mariadb_galera_active'] == false {
+        $restart_mariadb = true
+      }else{
+        $restart_mariadb = false
+      }
+    }
+
+    # prepare master cluster member
+    if $::facts['mariadb_installed'] and $galera_master_bool{
+
+      # set restart option for mariadb
+      $restart_mariadb = false
+
+      # check if cluster is already bootstrapped
+      if $cluster_bootstrapped {
+
+        # set service manage/enabled to match $cluster_bootstrapped
+        $real_service_manage = true
+        $real_service_enabled = true
+
+        # set overrides
+        $merged_overrides = $merged_overrides_both
+
+      }else{
+
+        # set overrides
+        $merged_overrides = $merged_overrides_both
+
+        # bootstrap a cluster, as this is the first setup, mariadb should be active
+        if $::facts['mariadb_active'] {
+
+          # stop mariadb before bootstrapping
+          exec { 'stop_mariadb_for_bootstrap':
+            command => 'systemctl stop mariadb',
+            path    => ['/bin', '/usr/bin'],
+            require => Class['Mysql::server'],
+          }
+
+          # bootstrap galera cluster
+          # only run if the cluster is not already bootstrapped
+          exec { 'bootstrap_galera_new_cluster':
+            command => 'galera_new_cluster',
+            path    => ['/bin', '/usr/bin'],
+            require => Class['Mysql::server'],
+          }
+        }
+      }
+    }
+
+    # prepare for initial run, this will create a single-node mariadb host
+    if ! $::facts['mariadb_installed'] {
+      $merged_overrides = $merged_overrides_mysqld_only
+      $restart_mariadb = true
+    }
+
+    class { 'mysql::server':
+      root_password           => $root_password,
+      remove_default_accounts => true,
+      restart                 => $restart_mariadb,
+      service_manage          => $real_service_manage,
+      service_enabled         => $real_service_enabled,
+      package_manage          => $package_manage,
+      package_name            => $package_name,
+      override_options        => $merged_overrides,
+    }
+
+  }else{
+    notice("${title} requires the servers_array to have 3 or more, currently it is ${length($servers_array)}.")
+  }
+}

site/profiles/manifests/yum/mariadb.pp

@@ -0,0 +1,24 @@
+# Class: profiles::yum::mariadb
+#
+# This class manages the mariadb yum repository for the system.
+#
+class profiles::yum::mariadb (
+  String        $baseurl = 'http://repos.main.unkin.net',
+  String        $version = '11.2',
+  Enum[
+    'daily',
+    'weekly',
+    'monthly'
+  ]             $snapshot = 'daily',
+) {
+  $release = $facts['os']['release']['major']
+  $basearch = $facts['os']['architecture']
+
+  yumrepo { 'mariadb':
+    name    => 'mariadb',
+    descr   => 'mariadb repository',
+    target  => '/etc/yum.repos.d/mariadb.repo',
+    baseurl => "${baseurl}/mariadb/${version}/el${release}-${snapshot}/${basearch}/os/",
+    gpgkey  => "${baseurl}/mariadb/${version}/el${release}-${snapshot}/${basearch}/os/RPM-GPG-KEY-MariaDB",
+  }
+}

site/roles/manifests/infra/sql/galera.pp

@@ -2,4 +2,6 @@
 class roles::infra::sql::galera {
   include profiles::defaults
   include profiles::base
+  include profiles::base::datavol
+  include profiles::sql::galera_member
 }