feat: add minio profile

- add additional modules in Puppetfile - update puppetlabs-lvm to 2.1.0 - add facts.d base path to hieradata - add infra/storage and infra/storage/minio role data to hieradata - add new facts for minio setup status - add a static yaml minio-facts file to assist dynamic ruby facts - updated hiera with additional directories (country/{role,region})
2023-12-24 13:54:40 +11:00 · 2023-12-24 13:54:40 +11:00 · d8751ac6c8
commit d8751ac6c8
parent f260b09d49
15 changed files with 449 additions and 7 deletions
--- a/4
+++ b/4
@ -8,7 +8,7 @@ mod 'puppetlabs-concat', '9.0.0'
 mod 'puppetlabs-vcsrepo', '6.1.0'
 mod 'puppetlabs-yumrepo_core', '2.0.0'
 mod 'puppetlabs-apt', '9.1.0'
-mod 'puppetlabs-lvm', '2.0.3'
+mod 'puppetlabs-lvm', '2.1.0'
 mod 'puppetlabs-puppetdb', '7.13.0'
 mod 'puppetlabs-postgresql', '9.1.0'
 mod 'puppetlabs-firewall', '6.0.0'
@ -33,6 +33,8 @@ mod 'ghoneycutt-puppet', '3.3.0'
 mod 'saz-sudo', '8.0.0'
 mod 'dalen-puppetdbquery', '3.0.1'
 mod 'markt-galera', '3.1.0'
+mod 'kogitoapp-minio', '1.1.4'
+mod 'broadinstitute-certs', '3.0.1'

 mod 'bind',
  :git => 'https://git.unkin.net/unkinben/puppet-bind.git',
--- a/hiera.yaml
+++ b/hiera.yaml
@ -7,14 +7,28 @@ hierarchy:
  - name: Consolidated Data
    paths:
      - "nodes/%{trusted.certname}.yaml"
-      - "roles/%{::enc_role_tier1}.eyaml"
-      - "roles/%{::enc_role_tier1}.yaml"
-      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml"
-      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}.eyaml"
+      - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}.yaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}.eyaml"
+      - "country/%{::country}/roles/%{::enc_role_tier1}.yaml"
+      - "country/%{::country}/region/%{::region}.eyaml"
+      - "country/%{::country}/region/%{::region}.yaml"
+      - "country/%{::country}.eyaml"
+      - "country/%{::country}.yaml"
      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml"
      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml"
-      - "%{::enc_role_path}.eyaml"
-      - "%{::enc_role_path}.yaml"
+      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml"
+      - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml"
+      - "roles/%{::enc_role_tier1}.eyaml"
+      - "roles/%{::enc_role_tier1}.yaml"
      - "os/%{facts.os.name}/%{facts.os.name}%{facts.os.release.major}.yaml"
      - "os/%{facts.os.name}/all_releases.yaml"
      - "common.eyaml"
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@ -7,6 +7,8 @@ lookup_options:
    merge:
      strategy: deep

+facts_path: '/opt/puppetlabs/facter/facts.d'
+
 profiles::ntp::client::ntp_role: 'roles::infra::ntp::server'
 profiles::ntp::client::peers:
  - 0.pool.ntp.org
--- a/hieradata/country/au/region/drw1/infra/storage/minio.yaml
+++ b/hieradata/country/au/region/drw1/infra/storage/minio.yaml
@ -0,0 +1,6 @@
+---
+profiles::minio::server::minio_members: 5
+profiles::minio::server::blockdev:
+  - /dev/sda
+  - /dev/sdb
+profiles::minio::server::minio_storage_class: 'EC:2'
--- a/hieradata/roles/infra/storage/minio.eyaml
+++ b/hieradata/roles/infra/storage/minio.eyaml
@ -0,0 +1,2 @@
+---
+profiles::minio::server::minio_root_pass: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAcAGh4K8P/bOwHs7FEAssBcYgtT+FlkMW4/jpJf230sbOh0jswvCl0woQMMw+AIpkXNJ//YcmDBkhdE92RCK8C4Xi/2nkdWjPt9FQwuT47BhAKISjunRs9R61dKj5aOwAlTQ3lNtsQsknGz17AMTyPEGQC9SnPxYirLRr9VgJX/EKPjl7M2LbkZTJChwIE6IiT+LSzye7YgpkJ7O6h4jNIp5ryWaUqSUfooYjqHc1zl4Bs9ZfyY1K/CWCTIbtd4hY1ZlskRlVa9yA0cWhsufV0gw43RA/bCAJPowLc64bZ4XlLx9Fy0qHKjTCDRLzysUoq0QjIR2Ulf1TkcCJAVLwFDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC3+P+RW/JoQemkVJE/mpAngDAw1JpFkBvLj4AlbJePvpnG+fFN8coOE+5N94NgGd9Gtl2NZt/g5x/7xFHS28cSlIg=]
--- a/hieradata/roles/infra/storage/minio.yaml
+++ b/hieradata/roles/infra/storage/minio.yaml
@ -0,0 +1,9 @@
+---
+profiles::minio::server::minio_members_role: roles::infra::storage::minio
+profiles::minio::server::minio_root_user: admin
+profiles::minio::server::minio_opts:
+  - '--anonymous'
+profiles::minio::server::minio_members_lookup: true
+profiles::minio::server::version: 'RELEASE.2023-12-20T01-00-02Z'
+profiles::minio::server::checksum: '09fafaf399885b4912bafda6fa03fc4ccbc39ec45e17239677217317915d6aeb'
+profiles::minio::server::checksum_type: 'sha256'
--- a/site/profiles/lib/facter/minio_datadirs_initialised.rb
+++ b/site/profiles/lib/facter/minio_datadirs_initialised.rb
@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# check all datadirs for minio are initialised
+
+require 'yaml'
+Facter.add('minio_datadirs_initialised') do
+  setcode do
+    yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml'
+
+    # check if the YAML file exists first
+    next false unless File.exist?(yaml_file_path)
+
+    minio_facts = YAML.load_file(yaml_file_path)
+    dev_count = minio_facts['minio_blockdev_count']
+    datadir = minio_facts['minio_datadir']
+
+    # check datadir if no blockdevices are used, otherwise check the store locations
+    if dev_count.zero?
+      Dir.exist?(datadir)
+    else
+      (1..dev_count).all? do |number|
+        Dir.exist?("#{datadir}/store#{number}")
+      end
+    end
+  end
+end
--- a/site/profiles/lib/facter/minio_group_exists.rb
+++ b/site/profiles/lib/facter/minio_group_exists.rb
@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# check that the minio group exists
+
+require 'yaml'
+Facter.add('minio_group_exists') do
+  setcode do
+    yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml'
+
+    # check if the YAML file exists first
+    next false unless File.exist?(yaml_file_path)
+
+    minio_facts = YAML.load_file(yaml_file_path)
+    group_name = minio_facts['minio_group']
+
+    group_exists = system("getent group #{group_name} >/dev/null 2>&1")
+
+    group_exists
+  end
+end
--- a/site/profiles/lib/facter/minio_pool_dns.rb
+++ b/site/profiles/lib/facter/minio_pool_dns.rb
@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'facter'
+require 'yaml'
+
+Facter.add('minio_pool_dns') do
+  setcode do
+    yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml'
+
+    # check if the YAML file exists
+    next {} unless File.exist?(yaml_file_path)
+
+    # load data from YAML
+    data = YAML.load_file(yaml_file_path)
+    minio_members = data['minio_members']
+    minio_region = data['minio_region']
+    minio_pool = data['minio_pool']
+    domain = Facter.value(:networking)['domain']
+
+    # create result hash
+    result = {}
+
+    # Check CNAME for each node_id from 1 to minio_members
+    (1..minio_members).each do |node_id|
+      cname_target = "#{minio_region}-#{minio_pool}-#{node_id}.#{domain}"
+      command = "host #{cname_target} > /dev/null 2>&1"
+
+      # Using system method to execute the command
+      # It returns true if the command gives exit status 0 (success), otherwise false
+      result[cname_target] = system(command)
+    end
+
+    result
+  end
+end
--- a/site/profiles/lib/facter/minio_user_exists.rb
+++ b/site/profiles/lib/facter/minio_user_exists.rb
@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# check that the minio user exists
+
+require 'yaml'
+Facter.add('minio_user_exists') do
+  setcode do
+    yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml'
+
+    # check if the YAML file exists first
+    next false unless File.exist?(yaml_file_path)
+
+    minio_facts = YAML.load_file(yaml_file_path)
+    user_name = minio_facts['minio_user']
+
+    user_exists = system("id #{user_name} >/dev/null 2>&1")
+
+    user_exists
+  end
+end
--- a/site/profiles/manifests/minio/server.pp
+++ b/site/profiles/manifests/minio/server.pp
@ -0,0 +1,208 @@
+# profiles::minio::server
+class profiles::minio::server (
+  String    $minio_root_user,
+  String    $minio_root_pass,
+  Array     $minio_opts = [],
+  Boolean   $minio_members_lookup = false,
+  String    $minio_members_role = undef,
+  Integer   $minio_members = undef,
+  Array     $minio_servers = [],
+  String    $minio_storage_class = 'EC:2',
+  String    $version = 'RELEASE.2023-12-20T01-00-02Z',
+  String    $checksum = '09fafaf399885b4912bafda6fa03fc4ccbc39ec45e17239677217317915d6aeb',
+  String    $checksum_type = 'sha256',
+  String    $owner = 'minio',
+  String    $group = 'minio',
+  Stdlib::Fqdn            $url_domain = $::facts['networking']['domain'],
+  Enum['http', 'https']   $url_scheme = 'http',
+  Enum['puppet', undef]          $cert_type = 'puppet',
+  Array[String[0]]        $blockdev = [],
+  Stdlib::Port            $listen_port = 9000,
+  Stdlib::IP::Address     $listen_addr = $::facts['networking']['ip'],
+  Stdlib::AbsolutePath    $datadir = '/data/minio',
+  Stdlib::AbsolutePath    $confdir = '/etc/minio',
+  Stdlib::AbsolutePath    $homedir = '/var/lib/minio',
+  Stdlib::AbsolutePath    $bindir = '/opt/minio',
+) {
+
+  # create the region string
+  $minio_region = "${::facts['country']}-${::facts['region']}"
+
+  # count the block devices
+  $blockdev_count = count($blockdev)
+
+  # create minio static facts, which are used by pre-compile facts
+  file { "${lookup('facts_path')}/minio_facts.yaml":
+    ensure  => file,
+    content => template('profiles/minio/minio_facts.yaml.erb'),
+  }
+
+  # create the user if its not yet initialised, if it is initialised, let the minio::server class manage
+  # manage the resource. This is done so that the user/group exist before attempting to create the data-
+  # directories.
+  if ! $::facts['minio_user_exists'] {
+    class {'minio::server::user':
+      manage_user  => true,
+      manage_group => true,
+      manage_home  => true,
+      owner        => $owner,
+      group        => $group,
+      home         => $homedir,
+    }
+  }
+
+  # create the datadir
+  if ! $::facts['minio_datadirs_initialised'] {
+
+    # create the datadir root
+    exec { "mkdir_p_${datadir}":
+      command => "mkdir -p '${datadir}'",
+      unless  => "test -d '${datadir}'",
+      path    => '/bin:/usr/bin',
+    }
+
+    # create te datavol's if blockdev's are listed
+    $blockdev.each |Integer $index, String $device| {
+      $id = $index + 1
+      profiles::storage::datavol {"${::facts['networking']['fqdn']}_${device}":
+        fstype  => 'xfs',
+        vg      => "minio_vg${id}",
+        pv      => $device,
+        lv      => "store${id}",
+        owner   => $owner,
+        group   => $group,
+        mount   => "${datadir}/store${id}",
+        require => Exec["mkdir_p_${datadir}"],
+      }
+    }
+  }
+
+  # copy puppet certs to /etc/pki/tls/puppet
+  include profiles::pki::puppetcerts
+
+  # create the cert configuration hash
+  if $cert_type == 'puppet' {
+    $cert_conf = {
+      'source_path'      => '/etc/pki/tls/puppet',
+      'source_cert_name' => $::facts['networking']['fqdn'],
+      'source_key_name'  => $::facts['networking']['fqdn'],
+    }
+  }
+
+  # if lookup is enabled, find all the hosts in the specified role and create the servers_array
+  if $minio_members_lookup {
+
+    # check that the role is also set
+    unless !($minio_members_role == undef) {
+      fail("minio_members_role must be provided for ${title} when minio_members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    #$servers_array = sort(query_nodes("enc_role='${minio_members_role}'", 'networking.fqdn'))
+    $servers_array = sort(query_nodes("enc_role='${minio_members_role}' and minio_region='${minio_region}'", 'networking.fqdn'))
+
+  # else use provided array from params
+  }else{
+    $servers_array = $minio_servers
+  }
+
+  # iterate through the servers_array and find the nodeid for each host
+  $servers_array.each |Integer $index, String $server| {
+    $id = $index + 1
+    if $::facts['networking']['fqdn'] == $server {
+      $nodeid = $id
+      if $::facts['minio_pool'] != undef {
+
+        # create a cname which is used to create a sequential group of names for distributed minio pool
+        profiles::dns::record { "${minio_region}-${::facts['minio_pool']}-${nodeid}.${::facts['networking']['domain']}_CNAME":
+          value  => $::facts['networking']['hostname'],
+          type   => 'CNAME',
+          record => "${minio_region}-${::facts['minio_pool']}-${nodeid}.${::facts['networking']['domain']}.",
+          zone   => $::facts['networking']['domain'],
+          order  => 10,
+        }
+      }
+    }
+  }
+
+  # wait until all expected servers in the pool have reported into puppet
+  if count($servers_array) == $::facts['minio_members'] and $::facts['minio_pool'] != undef {
+
+    # if datadirs and user have been initialised, prepare configuration.
+    if $::facts['minio_datadirs_initialised'] and $::facts['minio_user_exists'] {
+
+      # join the minio_opts
+      $options = join($minio_opts, ' ')
+
+      # create vars for $deployment_definition, others used below are params
+      $url_location = "${minio_region}-${::facts['minio_pool']}"
+      $url_servers = "1...${count($servers_array)}"
+
+      # create the deployment definition line
+        # >= 1 : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio/store{1...4}
+        # == 1 : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio/store1
+        # else : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio
+      if $blockdev_count >= 1 {
+        $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}/store{1...${blockdev_count}}"
+      }elsif $blockdev_count == 1 {
+        $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}/store1"
+      }else{
+        $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}"
+      }
+
+      # create the configuration hash
+      $configuration = {
+        'MINIO_ROOT_USER'              => $minio_root_user,
+        'MINIO_ROOT_PASSWORD'          => $minio_root_pass.unwrap,
+        'MINIO_REGION_NAME'            => $minio_region,
+        'MINIO_OPTS'                   => "\'${options}\'",
+        'MINIO_DEPLOYMENT_DEFINITION'  => $deployment_definition,
+        'MINIO_STORAGE_CLASS_STANDARD' => $minio_storage_class,
+      }
+    }
+  }
+
+  # check all the expected DNS CNAME records do not exist
+  $all_dns_exist = $::facts['minio_pool_dns'].all |String $cname, Boolean $exists| { $exists }
+
+  # create the minio server if all dns records exist
+  if $all_dns_exist {
+    class { 'minio::server::install':
+      package_ensure                 => 'present',
+      owner                          => $owner,
+      group                          => $group,
+      base_url                       => 'https://dl.minio.io/server/minio/release',
+      version                        => $version,
+      checksum                       => $checksum,
+      checksum_type                  => $checksum_type,
+      configuration_directory        => $confdir,
+      installation_directory         => $bindir,
+      storage_root                   => $datadir,
+      listen_ip                      => $listen_addr,
+      listen_port                    => $listen_port,
+      manage_service                 => true,
+      service_template               => 'profiles/minio/minio.service.erb',
+      service_provider               => 'systemd',
+      cert_directory                 => "${confdir}/certs",
+      custom_configuration_file_path => '/etc/default/minio',
+    }
+
+    class { 'minio::server::config':
+      owner                          => $owner,
+      group                          => $group,
+      configuration_directory        => $confdir,
+      installation_directory         => $bindir,
+      storage_root                   => $datadir,
+      configuration                  => $configuration,
+      custom_configuration_file_path => '/etc/default/minio',
+      require                        => Class['Minio::Server::Install'],
+    }
+
+    class { 'minio::server::service':
+      manage_service   => true,
+      service_provider => 'systemd',
+      service_ensure   => 'running',
+      require          => Class['Minio::Server::Config'],
+    }
+  }
+}
--- a/site/profiles/manifests/pki/puppetcerts.pp
+++ b/site/profiles/manifests/pki/puppetcerts.pp
@ -0,0 +1,42 @@
+# profiles::pki::puppetcerts
+class profiles::pki::puppetcerts {
+
+  # Define the directory
+  file { '/etc/pki/tls/puppet':
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  # Copy the CA certificate
+  file { '/etc/pki/tls/puppet/ca.pem':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => '/etc/puppetlabs/puppet/ssl/certs/ca.pem',
+    require => File['/etc/pki/tls/puppet'],
+  }
+
+  # Copy the private key
+  file { "/etc/pki/tls/puppet/${facts['networking']['fqdn']}.key":
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0600',
+    source  => "/etc/puppetlabs/puppet/ssl/private_keys/${facts['networking']['fqdn']}.pem",
+    require => File['/etc/pki/tls/puppet'],
+  }
+
+  # Copy the certificate
+  $cert = "/etc/puppetlabs/puppet/ssl/certs/${facts['networking']['fqdn']}.pem"
+  file { "/etc/pki/tls/puppet/${facts['networking']['fqdn']}.crt":
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => "/etc/puppetlabs/puppet/ssl/certs/${facts['networking']['fqdn']}.pem",
+    require => File['/etc/pki/tls/puppet'],
+  }
+}
--- a/site/profiles/templates/minio/minio.service.erb
+++ b/site/profiles/templates/minio/minio.service.erb
@ -0,0 +1,38 @@
+[Unit]
+Description=Minio
+Documentation=https://docs.minio.io
+Wants=network-online.target
+After=network-online.target
+After=syslog.target network.target
+AssertFileIsExecutable=<%= @installation_directory %>/minio
+
+[Service]
+WorkingDirectory=<%= @installation_directory %>
+
+User=<%= @owner %>
+Group=<%= @group %>
+
+PermissionsStartOnly=true
+
+EnvironmentFile=<%= @configuration_file_path %>
+ExecStart=<%= @installation_directory %>/minio server $MINIO_OPTS --address <%= @listen_ip %>:<%= @listen_port %> $MINIO_DEPLOYMENT_DEFINITION
+
+StandardOutput=journal
+StandardError=inherit
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Disable timeout logic and wait until process is stopped
+TimeoutStopSec=0
+
+# SIGTERM signal is used to stop Minio
+KillSignal=SIGTERM
+
+SendSIGKILL=no
+
+SuccessExitStatus=0
+
+[Install]
+WantedBy=multi-user.target
+
--- a/site/profiles/templates/minio/minio_facts.yaml.erb
+++ b/site/profiles/templates/minio/minio_facts.yaml.erb
@ -0,0 +1,17 @@
+# minio_facts.yaml
+minio_user: '<%= @owner %>'
+minio_group: '<%= @group %>'
+minio_pool: '<%= @minio_pool %>'
+minio_datadir: '<%= @datadir %>'
+minio_confdir: '<%= @confdir %>'
+minio_homedir: '<%= @homedir %>'
+minio_bindir: '<%= @bindir %>'
+minio_region: '<%= @minio_region %>'
+minio_members: <%= @minio_members %>
+minio_blockdev_count: <%= @blockdev_count %>
+<% unless @blockdev.empty? -%>
+minio_blockdevs:
+<% @blockdev.each do |device| -%>
+  - '<%= device %>'
+<% end -%>
+<% end -%>
--- a/site/roles/manifests/infra/storage/minio.pp
+++ b/site/roles/manifests/infra/storage/minio.pp
@ -2,4 +2,5 @@
 class roles::infra::storage::minio {
  include profiles::defaults
  include profiles::base
+  include profiles::minio::server
 }