diff --git a/Puppetfile b/Puppetfile index c197bde..4654fa0 100644 --- a/Puppetfile +++ b/Puppetfile @@ -8,7 +8,7 @@ mod 'puppetlabs-concat', '9.0.0' mod 'puppetlabs-vcsrepo', '6.1.0' mod 'puppetlabs-yumrepo_core', '2.0.0' mod 'puppetlabs-apt', '9.1.0' -mod 'puppetlabs-lvm', '2.0.3' +mod 'puppetlabs-lvm', '2.1.0' mod 'puppetlabs-puppetdb', '7.13.0' mod 'puppetlabs-postgresql', '9.1.0' mod 'puppetlabs-firewall', '6.0.0' @@ -33,6 +33,8 @@ mod 'ghoneycutt-puppet', '3.3.0' mod 'saz-sudo', '8.0.0' mod 'dalen-puppetdbquery', '3.0.1' mod 'markt-galera', '3.1.0' +mod 'kogitoapp-minio', '1.1.4' +mod 'broadinstitute-certs', '3.0.1' mod 'bind', :git => 'https://git.unkin.net/unkinben/puppet-bind.git', diff --git a/hiera.yaml b/hiera.yaml index bccbba6..b763ee3 100644 --- a/hiera.yaml +++ b/hiera.yaml @@ -7,14 +7,28 @@ hierarchy: - name: Consolidated Data paths: - "nodes/%{trusted.certname}.yaml" - - "roles/%{::enc_role_tier1}.eyaml" - - "roles/%{::enc_role_tier1}.yaml" - - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml" - - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}.eyaml" + - "country/%{::country}/region/%{::region}/%{::enc_role_tier1}.yaml" + - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml" + - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml" + - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml" + - "country/%{::country}/roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml" + - "country/%{::country}/roles/%{::enc_role_tier1}.eyaml" + - "country/%{::country}/roles/%{::enc_role_tier1}.yaml" + - "country/%{::country}/region/%{::region}.eyaml" + - "country/%{::country}/region/%{::region}.yaml" + - "country/%{::country}.eyaml" + - "country/%{::country}.yaml" - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.eyaml" - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}/%{::enc_role_tier3}.yaml" - - "%{::enc_role_path}.eyaml" - - "%{::enc_role_path}.yaml" + - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.eyaml" + - "roles/%{::enc_role_tier1}/%{::enc_role_tier2}.yaml" + - "roles/%{::enc_role_tier1}.eyaml" + - "roles/%{::enc_role_tier1}.yaml" - "os/%{facts.os.name}/%{facts.os.name}%{facts.os.release.major}.yaml" - "os/%{facts.os.name}/all_releases.yaml" - "common.eyaml" diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 79f8edb..dc4b711 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -7,6 +7,8 @@ lookup_options: merge: strategy: deep +facts_path: '/opt/puppetlabs/facter/facts.d' + profiles::ntp::client::ntp_role: 'roles::infra::ntp::server' profiles::ntp::client::peers: - 0.pool.ntp.org diff --git a/hieradata/country/au/region/drw1/infra/storage/minio.yaml b/hieradata/country/au/region/drw1/infra/storage/minio.yaml new file mode 100644 index 0000000..1b428c1 --- /dev/null +++ b/hieradata/country/au/region/drw1/infra/storage/minio.yaml @@ -0,0 +1,6 @@ +--- +profiles::minio::server::minio_members: 5 +profiles::minio::server::blockdev: + - /dev/sda + - /dev/sdb +profiles::minio::server::minio_storage_class: 'EC:2' diff --git a/hieradata/roles/infra/storage/minio.eyaml b/hieradata/roles/infra/storage/minio.eyaml new file mode 100644 index 0000000..08e0b2d --- /dev/null +++ b/hieradata/roles/infra/storage/minio.eyaml @@ -0,0 +1,2 @@ +--- +profiles::minio::server::minio_root_pass: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAcAGh4K8P/bOwHs7FEAssBcYgtT+FlkMW4/jpJf230sbOh0jswvCl0woQMMw+AIpkXNJ//YcmDBkhdE92RCK8C4Xi/2nkdWjPt9FQwuT47BhAKISjunRs9R61dKj5aOwAlTQ3lNtsQsknGz17AMTyPEGQC9SnPxYirLRr9VgJX/EKPjl7M2LbkZTJChwIE6IiT+LSzye7YgpkJ7O6h4jNIp5ryWaUqSUfooYjqHc1zl4Bs9ZfyY1K/CWCTIbtd4hY1ZlskRlVa9yA0cWhsufV0gw43RA/bCAJPowLc64bZ4XlLx9Fy0qHKjTCDRLzysUoq0QjIR2Ulf1TkcCJAVLwFDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC3+P+RW/JoQemkVJE/mpAngDAw1JpFkBvLj4AlbJePvpnG+fFN8coOE+5N94NgGd9Gtl2NZt/g5x/7xFHS28cSlIg=] diff --git a/hieradata/roles/infra/storage/minio.yaml b/hieradata/roles/infra/storage/minio.yaml new file mode 100644 index 0000000..62505fb --- /dev/null +++ b/hieradata/roles/infra/storage/minio.yaml @@ -0,0 +1,9 @@ +--- +profiles::minio::server::minio_members_role: roles::infra::storage::minio +profiles::minio::server::minio_root_user: admin +profiles::minio::server::minio_opts: + - '--anonymous' +profiles::minio::server::minio_members_lookup: true +profiles::minio::server::version: 'RELEASE.2023-12-20T01-00-02Z' +profiles::minio::server::checksum: '09fafaf399885b4912bafda6fa03fc4ccbc39ec45e17239677217317915d6aeb' +profiles::minio::server::checksum_type: 'sha256' diff --git a/site/profiles/lib/facter/minio_datadirs_initialised.rb b/site/profiles/lib/facter/minio_datadirs_initialised.rb new file mode 100644 index 0000000..17180ab --- /dev/null +++ b/site/profiles/lib/facter/minio_datadirs_initialised.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# check all datadirs for minio are initialised + +require 'yaml' +Facter.add('minio_datadirs_initialised') do + setcode do + yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml' + + # check if the YAML file exists first + next false unless File.exist?(yaml_file_path) + + minio_facts = YAML.load_file(yaml_file_path) + dev_count = minio_facts['minio_blockdev_count'] + datadir = minio_facts['minio_datadir'] + + # check datadir if no blockdevices are used, otherwise check the store locations + if dev_count.zero? + Dir.exist?(datadir) + else + (1..dev_count).all? do |number| + Dir.exist?("#{datadir}/store#{number}") + end + end + end +end diff --git a/site/profiles/lib/facter/minio_group_exists.rb b/site/profiles/lib/facter/minio_group_exists.rb new file mode 100644 index 0000000..14c322a --- /dev/null +++ b/site/profiles/lib/facter/minio_group_exists.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# check that the minio group exists + +require 'yaml' +Facter.add('minio_group_exists') do + setcode do + yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml' + + # check if the YAML file exists first + next false unless File.exist?(yaml_file_path) + + minio_facts = YAML.load_file(yaml_file_path) + group_name = minio_facts['minio_group'] + + group_exists = system("getent group #{group_name} >/dev/null 2>&1") + + group_exists + end +end diff --git a/site/profiles/lib/facter/minio_pool_dns.rb b/site/profiles/lib/facter/minio_pool_dns.rb new file mode 100644 index 0000000..6ccf804 --- /dev/null +++ b/site/profiles/lib/facter/minio_pool_dns.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'facter' +require 'yaml' + +Facter.add('minio_pool_dns') do + setcode do + yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml' + + # check if the YAML file exists + next {} unless File.exist?(yaml_file_path) + + # load data from YAML + data = YAML.load_file(yaml_file_path) + minio_members = data['minio_members'] + minio_region = data['minio_region'] + minio_pool = data['minio_pool'] + domain = Facter.value(:networking)['domain'] + + # create result hash + result = {} + + # Check CNAME for each node_id from 1 to minio_members + (1..minio_members).each do |node_id| + cname_target = "#{minio_region}-#{minio_pool}-#{node_id}.#{domain}" + command = "host #{cname_target} > /dev/null 2>&1" + + # Using system method to execute the command + # It returns true if the command gives exit status 0 (success), otherwise false + result[cname_target] = system(command) + end + + result + end +end diff --git a/site/profiles/lib/facter/minio_user_exists.rb b/site/profiles/lib/facter/minio_user_exists.rb new file mode 100644 index 0000000..e1b3ef2 --- /dev/null +++ b/site/profiles/lib/facter/minio_user_exists.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# check that the minio user exists + +require 'yaml' +Facter.add('minio_user_exists') do + setcode do + yaml_file_path = '/opt/puppetlabs/facter/facts.d/minio_facts.yaml' + + # check if the YAML file exists first + next false unless File.exist?(yaml_file_path) + + minio_facts = YAML.load_file(yaml_file_path) + user_name = minio_facts['minio_user'] + + user_exists = system("id #{user_name} >/dev/null 2>&1") + + user_exists + end +end diff --git a/site/profiles/manifests/minio/server.pp b/site/profiles/manifests/minio/server.pp new file mode 100644 index 0000000..509e970 --- /dev/null +++ b/site/profiles/manifests/minio/server.pp @@ -0,0 +1,208 @@ +# profiles::minio::server +class profiles::minio::server ( + String $minio_root_user, + String $minio_root_pass, + Array $minio_opts = [], + Boolean $minio_members_lookup = false, + String $minio_members_role = undef, + Integer $minio_members = undef, + Array $minio_servers = [], + String $minio_storage_class = 'EC:2', + String $version = 'RELEASE.2023-12-20T01-00-02Z', + String $checksum = '09fafaf399885b4912bafda6fa03fc4ccbc39ec45e17239677217317915d6aeb', + String $checksum_type = 'sha256', + String $owner = 'minio', + String $group = 'minio', + Stdlib::Fqdn $url_domain = $::facts['networking']['domain'], + Enum['http', 'https'] $url_scheme = 'http', + Enum['puppet', undef] $cert_type = 'puppet', + Array[String[0]] $blockdev = [], + Stdlib::Port $listen_port = 9000, + Stdlib::IP::Address $listen_addr = $::facts['networking']['ip'], + Stdlib::AbsolutePath $datadir = '/data/minio', + Stdlib::AbsolutePath $confdir = '/etc/minio', + Stdlib::AbsolutePath $homedir = '/var/lib/minio', + Stdlib::AbsolutePath $bindir = '/opt/minio', +) { + + # create the region string + $minio_region = "${::facts['country']}-${::facts['region']}" + + # count the block devices + $blockdev_count = count($blockdev) + + # create minio static facts, which are used by pre-compile facts + file { "${lookup('facts_path')}/minio_facts.yaml": + ensure => file, + content => template('profiles/minio/minio_facts.yaml.erb'), + } + + # create the user if its not yet initialised, if it is initialised, let the minio::server class manage + # manage the resource. This is done so that the user/group exist before attempting to create the data- + # directories. + if ! $::facts['minio_user_exists'] { + class {'minio::server::user': + manage_user => true, + manage_group => true, + manage_home => true, + owner => $owner, + group => $group, + home => $homedir, + } + } + + # create the datadir + if ! $::facts['minio_datadirs_initialised'] { + + # create the datadir root + exec { "mkdir_p_${datadir}": + command => "mkdir -p '${datadir}'", + unless => "test -d '${datadir}'", + path => '/bin:/usr/bin', + } + + # create te datavol's if blockdev's are listed + $blockdev.each |Integer $index, String $device| { + $id = $index + 1 + profiles::storage::datavol {"${::facts['networking']['fqdn']}_${device}": + fstype => 'xfs', + vg => "minio_vg${id}", + pv => $device, + lv => "store${id}", + owner => $owner, + group => $group, + mount => "${datadir}/store${id}", + require => Exec["mkdir_p_${datadir}"], + } + } + } + + # copy puppet certs to /etc/pki/tls/puppet + include profiles::pki::puppetcerts + + # create the cert configuration hash + if $cert_type == 'puppet' { + $cert_conf = { + 'source_path' => '/etc/pki/tls/puppet', + 'source_cert_name' => $::facts['networking']['fqdn'], + 'source_key_name' => $::facts['networking']['fqdn'], + } + } + + # if lookup is enabled, find all the hosts in the specified role and create the servers_array + if $minio_members_lookup { + + # check that the role is also set + unless !($minio_members_role == undef) { + fail("minio_members_role must be provided for ${title} when minio_members_lookup is True") + } + + # if it is, find hosts, sort them so they dont cause changes every run + #$servers_array = sort(query_nodes("enc_role='${minio_members_role}'", 'networking.fqdn')) + $servers_array = sort(query_nodes("enc_role='${minio_members_role}' and minio_region='${minio_region}'", 'networking.fqdn')) + + # else use provided array from params + }else{ + $servers_array = $minio_servers + } + + # iterate through the servers_array and find the nodeid for each host + $servers_array.each |Integer $index, String $server| { + $id = $index + 1 + if $::facts['networking']['fqdn'] == $server { + $nodeid = $id + if $::facts['minio_pool'] != undef { + + # create a cname which is used to create a sequential group of names for distributed minio pool + profiles::dns::record { "${minio_region}-${::facts['minio_pool']}-${nodeid}.${::facts['networking']['domain']}_CNAME": + value => $::facts['networking']['hostname'], + type => 'CNAME', + record => "${minio_region}-${::facts['minio_pool']}-${nodeid}.${::facts['networking']['domain']}.", + zone => $::facts['networking']['domain'], + order => 10, + } + } + } + } + + # wait until all expected servers in the pool have reported into puppet + if count($servers_array) == $::facts['minio_members'] and $::facts['minio_pool'] != undef { + + # if datadirs and user have been initialised, prepare configuration. + if $::facts['minio_datadirs_initialised'] and $::facts['minio_user_exists'] { + + # join the minio_opts + $options = join($minio_opts, ' ') + + # create vars for $deployment_definition, others used below are params + $url_location = "${minio_region}-${::facts['minio_pool']}" + $url_servers = "1...${count($servers_array)}" + + # create the deployment definition line + # >= 1 : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio/store{1...4} + # == 1 : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio/store1 + # else : https://au-somewhere-1-pool1-{1...5}.example.domain/var/minio + if $blockdev_count >= 1 { + $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}/store{1...${blockdev_count}}" + }elsif $blockdev_count == 1 { + $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}/store1" + }else{ + $deployment_definition = "${url_scheme}://${url_location}-{${url_servers}}.${url_domain}:${listen_port}${datadir}" + } + + # create the configuration hash + $configuration = { + 'MINIO_ROOT_USER' => $minio_root_user, + 'MINIO_ROOT_PASSWORD' => $minio_root_pass.unwrap, + 'MINIO_REGION_NAME' => $minio_region, + 'MINIO_OPTS' => "\'${options}\'", + 'MINIO_DEPLOYMENT_DEFINITION' => $deployment_definition, + 'MINIO_STORAGE_CLASS_STANDARD' => $minio_storage_class, + } + } + } + + # check all the expected DNS CNAME records do not exist + $all_dns_exist = $::facts['minio_pool_dns'].all |String $cname, Boolean $exists| { $exists } + + # create the minio server if all dns records exist + if $all_dns_exist { + class { 'minio::server::install': + package_ensure => 'present', + owner => $owner, + group => $group, + base_url => 'https://dl.minio.io/server/minio/release', + version => $version, + checksum => $checksum, + checksum_type => $checksum_type, + configuration_directory => $confdir, + installation_directory => $bindir, + storage_root => $datadir, + listen_ip => $listen_addr, + listen_port => $listen_port, + manage_service => true, + service_template => 'profiles/minio/minio.service.erb', + service_provider => 'systemd', + cert_directory => "${confdir}/certs", + custom_configuration_file_path => '/etc/default/minio', + } + + class { 'minio::server::config': + owner => $owner, + group => $group, + configuration_directory => $confdir, + installation_directory => $bindir, + storage_root => $datadir, + configuration => $configuration, + custom_configuration_file_path => '/etc/default/minio', + require => Class['Minio::Server::Install'], + } + + class { 'minio::server::service': + manage_service => true, + service_provider => 'systemd', + service_ensure => 'running', + require => Class['Minio::Server::Config'], + } + } +} diff --git a/site/profiles/manifests/pki/puppetcerts.pp b/site/profiles/manifests/pki/puppetcerts.pp new file mode 100644 index 0000000..c3d2920 --- /dev/null +++ b/site/profiles/manifests/pki/puppetcerts.pp @@ -0,0 +1,42 @@ +# profiles::pki::puppetcerts +class profiles::pki::puppetcerts { + + # Define the directory + file { '/etc/pki/tls/puppet': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + + # Copy the CA certificate + file { '/etc/pki/tls/puppet/ca.pem': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0644', + source => '/etc/puppetlabs/puppet/ssl/certs/ca.pem', + require => File['/etc/pki/tls/puppet'], + } + + # Copy the private key + file { "/etc/pki/tls/puppet/${facts['networking']['fqdn']}.key": + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0600', + source => "/etc/puppetlabs/puppet/ssl/private_keys/${facts['networking']['fqdn']}.pem", + require => File['/etc/pki/tls/puppet'], + } + + # Copy the certificate + $cert = "/etc/puppetlabs/puppet/ssl/certs/${facts['networking']['fqdn']}.pem" + file { "/etc/pki/tls/puppet/${facts['networking']['fqdn']}.crt": + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0644', + source => "/etc/puppetlabs/puppet/ssl/certs/${facts['networking']['fqdn']}.pem", + require => File['/etc/pki/tls/puppet'], + } +} diff --git a/site/profiles/templates/minio/minio.service.erb b/site/profiles/templates/minio/minio.service.erb new file mode 100644 index 0000000..efaddbe --- /dev/null +++ b/site/profiles/templates/minio/minio.service.erb @@ -0,0 +1,38 @@ +[Unit] +Description=Minio +Documentation=https://docs.minio.io +Wants=network-online.target +After=network-online.target +After=syslog.target network.target +AssertFileIsExecutable=<%= @installation_directory %>/minio + +[Service] +WorkingDirectory=<%= @installation_directory %> + +User=<%= @owner %> +Group=<%= @group %> + +PermissionsStartOnly=true + +EnvironmentFile=<%= @configuration_file_path %> +ExecStart=<%= @installation_directory %>/minio server $MINIO_OPTS --address <%= @listen_ip %>:<%= @listen_port %> $MINIO_DEPLOYMENT_DEFINITION + +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop Minio +KillSignal=SIGTERM + +SendSIGKILL=no + +SuccessExitStatus=0 + +[Install] +WantedBy=multi-user.target + diff --git a/site/profiles/templates/minio/minio_facts.yaml.erb b/site/profiles/templates/minio/minio_facts.yaml.erb new file mode 100644 index 0000000..bc205b0 --- /dev/null +++ b/site/profiles/templates/minio/minio_facts.yaml.erb @@ -0,0 +1,17 @@ +# minio_facts.yaml +minio_user: '<%= @owner %>' +minio_group: '<%= @group %>' +minio_pool: '<%= @minio_pool %>' +minio_datadir: '<%= @datadir %>' +minio_confdir: '<%= @confdir %>' +minio_homedir: '<%= @homedir %>' +minio_bindir: '<%= @bindir %>' +minio_region: '<%= @minio_region %>' +minio_members: <%= @minio_members %> +minio_blockdev_count: <%= @blockdev_count %> +<% unless @blockdev.empty? -%> +minio_blockdevs: +<% @blockdev.each do |device| -%> + - '<%= device %>' +<% end -%> +<% end -%> diff --git a/site/roles/manifests/infra/storage/minio.pp b/site/roles/manifests/infra/storage/minio.pp index 252b35e..72411e8 100644 --- a/site/roles/manifests/infra/storage/minio.pp +++ b/site/roles/manifests/infra/storage/minio.pp @@ -2,4 +2,5 @@ class roles::infra::storage::minio { include profiles::defaults include profiles::base + include profiles::minio::server }