From f04c74bd4d809c854473285395cecc746e699989 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Wed, 17 Apr 2024 18:23:33 +1000 Subject: [PATCH] feat: manage proxmox nodes - change /etc/hosts to meet proxmox requirements - add proxmox node role - add init, params, repo, install, clusterjoin classes --- Puppetfile | 3 +- hieradata/common.eyaml | 1 + hieradata/common.yaml | 1 + .../country/au/region/syd1/infra/proxmox.yaml | 8 ++ .../nodes/prodnxsr0001.main.unkin.net.yaml | 5 ++ .../nodes/prodnxsr0002.main.unkin.net.yaml | 4 + .../nodes/prodnxsr0003.main.unkin.net.yaml | 4 + .../nodes/prodnxsr0004.main.unkin.net.yaml | 2 + .../nodes/prodnxsr0005.main.unkin.net.yaml | 2 + .../nodes/prodnxsr0006.main.unkin.net.yaml | 2 + .../nodes/prodnxsr0007.main.unkin.net.yaml | 2 + .../nodes/prodnxsr0008.main.unkin.net.yaml | 2 + hieradata/roles/infra/proxmox.yaml | 7 ++ modules/libs/lib/facter/is_pveceph_mgr.rb | 10 +++ modules/libs/lib/facter/is_pveceph_mon.rb | 10 +++ modules/libs/lib/facter/pve_ceph_config.rb | 34 +++++++++ .../libs/lib/facter/pve_ceph_initialised.rb | 10 +++ modules/libs/lib/facter/pve_cluster.rb | 28 +++++++ modules/libs/lib/facter/pve_cluster_member.rb | 21 ++++++ modules/libs/lib/facter/pve_nodelist.rb | 35 +++++++++ modules/libs/lib/facter/pve_nodes_active.rb | 17 +++++ site/profiles/manifests/proxmox/ceph.pp | 50 +++++++++++++ .../profiles/manifests/proxmox/clusterinit.pp | 41 ++++++++++ .../profiles/manifests/proxmox/clusterjoin.pp | 74 +++++++++++++++++++ site/profiles/manifests/proxmox/config.pp | 19 +++++ site/profiles/manifests/proxmox/init.pp | 16 ++++ site/profiles/manifests/proxmox/install.pp | 58 +++++++++++++++ site/profiles/manifests/proxmox/params.pp | 42 +++++++++++ site/profiles/manifests/proxmox/repos.pp | 37 ++++++++++ site/profiles/templates/base/hosts.erb | 7 +- .../templates/proxmox/join_pvecluster.erb | 11 +++ .../templates/proxmox/pve_facts.yaml.erb | 2 + site/roles/manifests/infra/proxmox/node.pp | 6 ++ 33 files changed, 564 insertions(+), 7 deletions(-) create mode 100644 hieradata/country/au/region/syd1/infra/proxmox.yaml create mode 100644 hieradata/nodes/prodnxsr0001.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0002.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0003.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0004.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0005.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0006.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0007.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0008.main.unkin.net.yaml create mode 100644 hieradata/roles/infra/proxmox.yaml create mode 100644 modules/libs/lib/facter/is_pveceph_mgr.rb create mode 100644 modules/libs/lib/facter/is_pveceph_mon.rb create mode 100644 modules/libs/lib/facter/pve_ceph_config.rb create mode 100644 modules/libs/lib/facter/pve_ceph_initialised.rb create mode 100644 modules/libs/lib/facter/pve_cluster.rb create mode 100644 modules/libs/lib/facter/pve_cluster_member.rb create mode 100644 modules/libs/lib/facter/pve_nodelist.rb create mode 100644 modules/libs/lib/facter/pve_nodes_active.rb create mode 100644 site/profiles/manifests/proxmox/ceph.pp create mode 100644 site/profiles/manifests/proxmox/clusterinit.pp create mode 100644 site/profiles/manifests/proxmox/clusterjoin.pp create mode 100644 site/profiles/manifests/proxmox/config.pp create mode 100644 site/profiles/manifests/proxmox/init.pp create mode 100644 site/profiles/manifests/proxmox/install.pp create mode 100644 site/profiles/manifests/proxmox/params.pp create mode 100644 site/profiles/manifests/proxmox/repos.pp create mode 100644 site/profiles/templates/proxmox/join_pvecluster.erb create mode 100644 site/profiles/templates/proxmox/pve_facts.yaml.erb create mode 100644 site/roles/manifests/infra/proxmox/node.pp diff --git a/Puppetfile b/Puppetfile index 9dc00c7..6d50a38 100644 --- a/Puppetfile +++ b/Puppetfile @@ -7,7 +7,7 @@ mod 'puppetlabs-inifile', '6.0.0' mod 'puppetlabs-concat', '9.0.0' mod 'puppetlabs-vcsrepo', '6.1.0' mod 'puppetlabs-yumrepo_core', '2.0.0' -mod 'puppetlabs-apt', '9.1.0' +mod 'puppetlabs-apt', '9.4.0' mod 'puppetlabs-lvm', '2.1.0' mod 'puppetlabs-puppetdb', '7.13.0' mod 'puppetlabs-postgresql', '9.1.0' @@ -17,6 +17,7 @@ mod 'puppetlabs-mysql', '15.0.0' mod 'puppetlabs-xinetd', '3.4.1' mod 'puppetlabs-haproxy', '8.0.0' mod 'puppetlabs-java', '10.1.2' +mod 'puppetlabs-reboot', '5.0.0' # puppet mod 'puppet-python', '7.0.0' diff --git a/hieradata/common.eyaml b/hieradata/common.eyaml index d6fee43..91e63a8 100644 --- a/hieradata/common.eyaml +++ b/hieradata/common.eyaml @@ -1,2 +1,3 @@ --- profiles::accounts::sysadmin::password: ENC[PKCS7,MIIBqQYJKoZIhvcNAQcDoIIBmjCCAZYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAoS7GyofFaXBNTWU+GtSiz4eCX/9j/sh3fDDRgOgNv1qpcQ87ZlTTenbHo9lxeURxKQ2HVVt7IsrBo/SC/WgipAKnliRkkIvo7nfAs+i+kEE8wakjAs0DcB4mhqtIZRuBkLG2Nay//DcG6cltVkbKEEKmKLMkDFZgTWreOZal8nDljpVe1S8QwtwP4/6hKTef5xsOnrisxuffWTXvwYJhj/VXrjdoH7EhtHGLybzEalglkVHEGft/WrrD/0bwJpmR0RegWI4HTsSvGiHgvf5DZJx8fXPZNPnicGtlfA9ccQPuVo17bY4Qf/WIc1A8Ssv4kHSbNIYJKRymI3UFb0Z4wzBsBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBBxDLb6pCGbittkcX6asd/gEBmMcUNupDjSECq5H09YA70eVwWWe0fBqxTxrr2cXCXtRKFvOk8SJmL0xHAWodaLN9+krTWHJcWbAK8JXEPC7rn] +profiles::accounts::root::password: ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAM79PRxeAZHrDcSm4eSFqU94/LjuSbdUmJWivX/Pa8GumoW2e/PT9nGHW3p98zHthMgCglk52PECQ+TBKjxr+9dTyNK5ePG6ZJEqSHNRqsPGm+kfQj/hlTmq8vOBaFM5GapD1iTHs5JFbGngI56swKBEVXW9+Z37BjQb2xJuyLsu5Bo/tA0BaOKuCtjq1a6E38bOX+nJ+YF1uZgV9ofAEh1YvkcTmnEWYXFRPWd7AaNcWn03V2pfhGqxc+xydak620I47P+FE+qIY72+aQ6tmLU3X9vyA1HLF2Tv572l4a2i+YIk6nAgQdi+hQKznqNL9M9YV+s1AcmcKLT7cfLrjsjA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCMWrdCWBQgtW3NOEpERwP+gBA3KDiqe4pQq6DwRfsEXQNZ] diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 9d5eab5..2f4d7dc 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -58,6 +58,7 @@ profiles::packages::install: - ccze - curl - dstat + - expect - gzip - git - htop diff --git a/hieradata/country/au/region/syd1/infra/proxmox.yaml b/hieradata/country/au/region/syd1/infra/proxmox.yaml new file mode 100644 index 0000000..5e07a1c --- /dev/null +++ b/hieradata/country/au/region/syd1/infra/proxmox.yaml @@ -0,0 +1,8 @@ +--- +profiles::proxmox::params::pve_members_role: 'roles::infra::proxmox::node' +profiles::proxmox::params::pve_kernel_version: '1.0.1' +profiles::proxmox::params::pve_kernel_release: '6.5.13-5-pve' +profiles::proxmox::params::pve_ceph_repos: true +profiles::proxmox::params::pve_ceph_release: 'reef' +profiles::proxmox::params::pve_ceph_install: true +profiles::proxmox::params::pve_ceph_network: '10.18.15.1/24' diff --git a/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml new file mode 100644 index 0000000..13bad49 --- /dev/null +++ b/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml @@ -0,0 +1,5 @@ +--- +profiles::proxmox::params::pve_clusterinit_master: true +profiles::proxmox::params::pve_ceph_mon: true +profiles::proxmox::params::pve_ceph_mgr: true +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml new file mode 100644 index 0000000..5fb387e --- /dev/null +++ b/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml @@ -0,0 +1,4 @@ +--- +profiles::proxmox::params::pve_ceph_mon: true +profiles::proxmox::params::pve_ceph_mgr: true +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml new file mode 100644 index 0000000..5fb387e --- /dev/null +++ b/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml @@ -0,0 +1,4 @@ +--- +profiles::proxmox::params::pve_ceph_mon: true +profiles::proxmox::params::pve_ceph_mgr: true +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml new file mode 100644 index 0000000..342f672 --- /dev/null +++ b/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml @@ -0,0 +1,2 @@ +--- +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml new file mode 100644 index 0000000..342f672 --- /dev/null +++ b/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml @@ -0,0 +1,2 @@ +--- +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml new file mode 100644 index 0000000..342f672 --- /dev/null +++ b/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml @@ -0,0 +1,2 @@ +--- +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml new file mode 100644 index 0000000..342f672 --- /dev/null +++ b/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml @@ -0,0 +1,2 @@ +--- +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml new file mode 100644 index 0000000..342f672 --- /dev/null +++ b/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml @@ -0,0 +1,2 @@ +--- +profiles::proxmox::params::pve_ceph_osd: true diff --git a/hieradata/roles/infra/proxmox.yaml b/hieradata/roles/infra/proxmox.yaml new file mode 100644 index 0000000..7a1b911 --- /dev/null +++ b/hieradata/roles/infra/proxmox.yaml @@ -0,0 +1,7 @@ +--- +sudo::configs: + ceph-smartctl: + priority: 20 + content: | + ceph ALL=NOPASSWD: /usr/sbin/smartctl -x --json=o /dev/* + ceph ALL=NOPASSWD: /usr/sbin/nvme * smart-log-add --json /dev/* diff --git a/modules/libs/lib/facter/is_pveceph_mgr.rb b/modules/libs/lib/facter/is_pveceph_mgr.rb new file mode 100644 index 0000000..cb1a243 --- /dev/null +++ b/modules/libs/lib/facter/is_pveceph_mgr.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('is_pveceph_mgr') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + system('pgrep -x ceph-mgr > /dev/null 2>&1') + end +end diff --git a/modules/libs/lib/facter/is_pveceph_mon.rb b/modules/libs/lib/facter/is_pveceph_mon.rb new file mode 100644 index 0000000..e32a312 --- /dev/null +++ b/modules/libs/lib/facter/is_pveceph_mon.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('is_pveceph_mon') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + system('pgrep -x ceph-mon > /dev/null 2>&1') + end +end diff --git a/modules/libs/lib/facter/pve_ceph_config.rb b/modules/libs/lib/facter/pve_ceph_config.rb new file mode 100644 index 0000000..e836a99 --- /dev/null +++ b/modules/libs/lib/facter/pve_ceph_config.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('ceph_global_config') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + config_file = '/etc/pve/ceph.conf' + config_hash = {} + in_global_section = false + + if File.exist?(config_file) + File.readlines(config_file).each do |line| + line.strip! + # Detect the [global] section and set flag + if line == '[global]' + in_global_section = true + next + end + + # Exit the loop once we're out of the global section + break if line.start_with?('[') && in_global_section + + # Parse key-value pairs if we are in the global section + if in_global_section && line.include?('=') + key, value = line.split('=', 2).map(&:strip) + config_hash[key] = value + end + end + end + + config_hash + end +end diff --git a/modules/libs/lib/facter/pve_ceph_initialised.rb b/modules/libs/lib/facter/pve_ceph_initialised.rb new file mode 100644 index 0000000..52c4c4e --- /dev/null +++ b/modules/libs/lib/facter/pve_ceph_initialised.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('pve_ceph_initialised') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + File.exist?('/etc/pve/ceph.conf') + end +end diff --git a/modules/libs/lib/facter/pve_cluster.rb b/modules/libs/lib/facter/pve_cluster.rb new file mode 100644 index 0000000..05efec1 --- /dev/null +++ b/modules/libs/lib/facter/pve_cluster.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('pve_cluster') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + conf_file = '/etc/pve/corosync.conf' + totem_details = {} + in_totem_section = false + + if File.exist?(conf_file) + File.foreach(conf_file) do |line| + if line =~ /^\s*totem\s*\{/ + in_totem_section = true + elsif line =~ /^\s*\}/ && in_totem_section + break + elsif in_totem_section && line =~ /^\s*(\w+):\s*(.+)$/ + key = Regexp.last_match(1).strip + value = Regexp.last_match(2).strip + totem_details[key] = value + end + end + end + + totem_details.empty? ? nil : totem_details + end +end diff --git a/modules/libs/lib/facter/pve_cluster_member.rb b/modules/libs/lib/facter/pve_cluster_member.rb new file mode 100644 index 0000000..602adf8 --- /dev/null +++ b/modules/libs/lib/facter/pve_cluster_member.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('pve_cluster_member') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + if Facter::Util::Resolution.which('pvesh') + cluster_status = `pvesh get /cluster/status --output-format json` + if cluster_status.empty? + false + else + require 'json' + status = JSON.parse(cluster_status) + !status.empty? + end + else + false + end + end +end diff --git a/modules/libs/lib/facter/pve_nodelist.rb b/modules/libs/lib/facter/pve_nodelist.rb new file mode 100644 index 0000000..4e16d81 --- /dev/null +++ b/modules/libs/lib/facter/pve_nodelist.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('pve_nodelist') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + conf_file = '/etc/pve/corosync.conf' + node_list = {} + current_node = nil + # rubocop:disable Metrics/BlockNesting + + if File.exist?(conf_file) + File.foreach(conf_file) do |line| + if line =~ /^\s*node\s*\{/ + current_node = {} + elsif line =~ /^\s*\}/ + if current_node + node_name = current_node['name'] + node_list[node_name] = current_node if node_name + current_node = nil + end + elsif current_node && line =~ /^\s*(\w+):\s*(.+)$/ + key = Regexp.last_match(1).strip + value = Regexp.last_match(2).strip + current_node[key] = value + end + end + end + + # rubocop:enable Metrics/BlockNesting + + node_list.empty? ? nil : node_list + end +end diff --git a/modules/libs/lib/facter/pve_nodes_active.rb b/modules/libs/lib/facter/pve_nodes_active.rb new file mode 100644 index 0000000..fade65d --- /dev/null +++ b/modules/libs/lib/facter/pve_nodes_active.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'facter' + +Facter.add('pve_nodes_active') do + confine enc_role: 'roles::infra::proxmox::node' + setcode do + if Facter::Util::Resolution.which('pvesh') + proxmox_nodes = `pvesh get /nodes --output-format json` + unless proxmox_nodes.empty? + require 'json' + nodes = JSON.parse(proxmox_nodes) + nodes.count + end + end + end +end diff --git a/site/profiles/manifests/proxmox/ceph.pp b/site/profiles/manifests/proxmox/ceph.pp new file mode 100644 index 0000000..3bff1b0 --- /dev/null +++ b/site/profiles/manifests/proxmox/ceph.pp @@ -0,0 +1,50 @@ +# profiles::proxmox::ceph +class profiles::proxmox::ceph { + + # include params class + include profiles::proxmox::params + + # localise some vars + $network = $profiles::proxmox::params::pve_ceph_network + $size = $profiles::proxmox::params::pve_ceph_size + $min_size = $profiles::proxmox::params::pve_ceph_minsize + + # install ceph if it is enabled + if $profiles::proxmox::params::pve_ceph_install { + + # initialise the cluster, but only on the clusterinit node and only if its not already initialised + if $profiles::proxmox::params::pve_clusterinit_master and ! $facts['pve_ceph_initialised']{ + exec { 'pveceph_init': + command => "/usr/bin/pveceph init --network ${network} --size ${size} --min_size ${min_size}", + user => 'root', + } + } + + if $facts['pve_ceph_initialised'] { + + # create monitors + if $profiles::proxmox::params::pve_ceph_mon { + + # only when its not already a monitor + if ! $facts['is_pveceph_mon'] { + exec { 'pveceph_mon': + command => '/usr/bin/pveceph mon create', + user => 'root', + } + } + } + + # create managers + if $profiles::proxmox::params::pve_ceph_mgr { + + # only when its not already a manager + if ! $facts['is_pveceph_mgr'] { + exec { 'pveceph_mgr': + command => '/usr/bin/pveceph mgr create', + user => 'root', + } + } + } + } + } +} diff --git a/site/profiles/manifests/proxmox/clusterinit.pp b/site/profiles/manifests/proxmox/clusterinit.pp new file mode 100644 index 0000000..65189b3 --- /dev/null +++ b/site/profiles/manifests/proxmox/clusterinit.pp @@ -0,0 +1,41 @@ +# profiles::proxmox::clusterinit +class profiles::proxmox::clusterinit { + + # include params class + include profiles::proxmox::params + + # localise some vars + $clusterinit_master = $profiles::proxmox::params::pve_clusterinit_master + $clustername = $profiles::proxmox::params::pve_cluster + $membersrole = $profiles::proxmox::params::pve_members_role + + # if this is the cluster master + if $clusterinit_master { + + # and its not a member of a cluster yet + if ! $facts['pve_cluster_member'] { + + # initialise a cluster + exec {'pve_init_cluster': + command => "/usr/bin/pvecm create ${clustername}", + unless => 'pvecm status', + timeout => 60, + } + } + } + + $servers_array = sort(query_nodes( + "enc_role='${membersrole}' and country='${facts['country']}' and region='${facts['region']}'", + 'networking.fqdn' + )) + + if ! $profiles::proxmox::params::pve_clusterinit_master { + if !empty($servers_array) { + notify { "Cluster ${profiles::proxmox::params::pve_cluster} detected, proceeding to join...": + } + } else { + notify { "No cluster flag found for ${profiles::proxmox::params::pve_cluster}, not attempting to join": + } + } + } +} diff --git a/site/profiles/manifests/proxmox/clusterjoin.pp b/site/profiles/manifests/proxmox/clusterjoin.pp new file mode 100644 index 0000000..7ab3ea5 --- /dev/null +++ b/site/profiles/manifests/proxmox/clusterjoin.pp @@ -0,0 +1,74 @@ +# profiles::proxmox::clusterjoin +class profiles::proxmox::clusterjoin { + + # include params class + include profiles::proxmox::params + + # localise some vars + $clusterinit_master = $profiles::proxmox::params::pve_clusterinit_master + $clustername = $profiles::proxmox::params::pve_cluster + $membersrole = $profiles::proxmox::params::pve_members_role + $root_password = $profiles::proxmox::params::root_password + + # query puppetdb for list of cluster members + $members_array = sort(query_nodes( + "enc_role='${membersrole}' and \ + country='${facts['country']}' and \ + region='${facts['region']}' and \ + pve_cluster.cluster_name='${clustername}'", + 'networking.fqdn' + )) + + # check if the pve kernerl is running + if $facts['kernelrelease'] == $profiles::proxmox::params::pve_kernel_release { + + # if this is the cluster master + if $clusterinit_master { + + # there are no cluster members in puppetdb + if empty($members_array) { + + # and this host isnt already in a cluster by itself + if ! $facts['pve_cluster'] { + + # initialise a cluster + exec {'pve_init_cluster': + command => "/usr/bin/pvecm create ${clustername}", + unless => 'pvecm status', + timeout => 60, + } + } + } + } + + # for non-masters + if ! $clusterinit_master { + + # if there are already members of the cluster + if !empty($members_array) { + + # and this host isnt already in a cluster + if ! $facts['pve_cluster'] { + + # create an expect script to join the cluster + file { '/usr/local/bin/join_pvecluster.expect': + ensure => file, + owner => 'root', + mode => '0755', + content => template('profiles/proxmox/join_pvecluster.erb'), + } + + exec { 'pve_join_cluster': + command => "/usr/local/bin/join_pvecluster.expect '${root_password.unwrap}' '${members_array[0]}'", + require => [File['/usr/local/bin/join_pvecluster.expect'], Package['expect']], + unless => "/usr/bin/pvesh nodes | grep -q '${facts['networking']['hostname']}'", + user => 'root', + } + } + } else { + notify { "No initialised cluster found for ${clustername}, not attempting to join": + } + } + } + } +} diff --git a/site/profiles/manifests/proxmox/config.pp b/site/profiles/manifests/proxmox/config.pp new file mode 100644 index 0000000..0edc8d1 --- /dev/null +++ b/site/profiles/manifests/proxmox/config.pp @@ -0,0 +1,19 @@ +# profiles::proxmox::config +class profiles::proxmox::config { + + # include params class + include profiles::proxmox::params + + # localise some vars + $clusterinit_master = $profiles::proxmox::params::pve_clusterinit_master + + # create pve_facts file + file {'/opt/puppetlabs/facter/facts.d/pve_facts.yaml': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0600', + content => template('profiles/proxmox/pve_facts.yaml.erb') + } + +} diff --git a/site/profiles/manifests/proxmox/init.pp b/site/profiles/manifests/proxmox/init.pp new file mode 100644 index 0000000..f7d769f --- /dev/null +++ b/site/profiles/manifests/proxmox/init.pp @@ -0,0 +1,16 @@ +# proxmox:: +class profiles::proxmox::init { + + #include profiles::proxmox::params + include profiles::proxmox::repos + include profiles::proxmox::install + include profiles::proxmox::clusterjoin + include profiles::proxmox::ceph + include profiles::proxmox::config + + Class['profiles::proxmox::repos'] + -> Class['profiles::proxmox::install'] + -> Class['profiles::proxmox::clusterjoin'] + -> Class['profiles::proxmox::ceph'] + -> Class['profiles::proxmox::config'] +} diff --git a/site/profiles/manifests/proxmox/install.pp b/site/profiles/manifests/proxmox/install.pp new file mode 100644 index 0000000..4cfdba2 --- /dev/null +++ b/site/profiles/manifests/proxmox/install.pp @@ -0,0 +1,58 @@ +# profiles::proxmox::install +class profiles::proxmox::install { + + # include params class + include profiles::proxmox::params + + # install the pve kernel + package { 'proxmox-default-kernel': + ensure => $profiles::proxmox::params::pve_kernel_version, + notify => Reboot['after_run'], + require => Apt::Source['proxmox'], + } + + # reboot into the new kernel + reboot { 'after_run': + apply => finished, + } + + if $facts['kernelrelease'] == $profiles::proxmox::params::pve_kernel_release { + + # install pve + ensure_packages($profiles::proxmox::params::pve_packages_install, { ensure => 'present', require => Apt::Source['proxmox']}) + + # remove the old linux kernel metapackage + ensure_packages($profiles::proxmox::params::pve_packages_remove, { ensure => 'absent' }) + + # install ceph package if requested + if $profiles::proxmox::params::pve_ceph_install { + ensure_packages($profiles::proxmox::params::pve_packages_ceph, { ensure => 'present', require => Apt::Source['ceph'] }) + } + + # cleanup the old kernel packages + exec { 'remove-linux-kernel-packages': + command => '/usr/bin/apt-get purge -y $(/usr/bin/dpkg --list | /bin/grep "linux-image-6.1" | /usr/bin/awk \'{ print $2 }\')', + onlyif => '/usr/bin/dpkg --list | /bin/grep -q "linux-image-6.1"', + path => ['/usr/bin', '/bin', '/sbin'], + refreshonly => true, + } + + # update grup + exec { 'update-grub': + command => '/usr/sbin/update-grub', + path => ['/usr/bin', '/bin', '/sbin'], + refreshonly => true, + } + + # update grub after removing kernel packages only + Exec['remove-linux-kernel-packages'] ~> Exec['update-grub'] + + # prepare for SDN + file_line { 'source-network-interfaces-d': + path => '/etc/network/interfaces', + line => 'source /etc/network/interfaces.d/*', + match => '^source /etc/network/interfaces.d/\*$', + append_on_no_match => true, + } + } +} diff --git a/site/profiles/manifests/proxmox/params.pp b/site/profiles/manifests/proxmox/params.pp new file mode 100644 index 0000000..2a4844e --- /dev/null +++ b/site/profiles/manifests/proxmox/params.pp @@ -0,0 +1,42 @@ +# profiles::proxmox::params +class profiles::proxmox::params ( + Sensitive[String] $root_password = Sensitive(lookup('profiles::accounts::root::password')), + String $pve_members_role = 'roles::infra::proxmox::node', + String $pve_kernel_version = '1.0.1', + String $pve_kernel_release = '6.5.13-5-pve', + String $pve_cluster = "${::facts['country']}-${::facts['region']}", + Boolean $pve_clusterinit_master = false, + Boolean $pve_ceph_repos = false, + Boolean $pve_ceph_install = false, + Boolean $pve_ceph_mon = false, + Boolean $pve_ceph_mgr = false, + Boolean $pve_ceph_osd = false, + String $pve_ceph_release = 'quincy', + Integer $pve_ceph_size = 3, + Integer $pve_ceph_minsize = 2, + Variant[ + Undef, + Stdlib::IP::Address::V4::CIDR + ] $pve_ceph_network = undef, + + Array $pve_packages_install = [ + 'proxmox-ve', + 'postfix', + 'open-iscsi', + 'frr-pythontools' + ], + Array $pve_packages_remove = [ + 'os-prober', + 'linux-image-amd64' + ], + Array $pve_packages_ceph = [ + 'ceph', + 'ceph-common', + 'ceph-fuse', + 'ceph-mds', + 'ceph-volume', + 'gdisk', + 'nvme-cli' + ] +){ +} diff --git a/site/profiles/manifests/proxmox/repos.pp b/site/profiles/manifests/proxmox/repos.pp new file mode 100644 index 0000000..0378fa4 --- /dev/null +++ b/site/profiles/manifests/proxmox/repos.pp @@ -0,0 +1,37 @@ +# profiles::proxmox::repos +class profiles::proxmox::repos { + + # include params class + include profiles::proxmox::params + + $codename = $facts['os']['distro']['codename'] + + exec { 'download-proxmox-gpg-key': + command => "/usr/bin/wget https://enterprise.proxmox.com/debian/proxmox-release-${codename}.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-${codename}.gpg", + creates => "/etc/apt/trusted.gpg.d/proxmox-release-${codename}.gpg", + path => ['/usr/bin', '/bin'], + require => File['/etc/apt/trusted.gpg.d/'], + } + + file { '/etc/apt/trusted.gpg.d/': + ensure => 'directory', + } + + apt::source { 'proxmox': + location => 'http://download.proxmox.com/debian/pve', + repos => 'pve-no-subscription', + include => { + src => false, + }, + } + + if $profiles::proxmox::params::pve_ceph_repos { + apt::source { 'ceph': + location => "http://download.proxmox.com/debian/ceph-${profiles::proxmox::params::pve_ceph_release}", + repos => 'no-subscription', + include => { + src => false, + }, + } + } +} diff --git a/site/profiles/templates/base/hosts.erb b/site/profiles/templates/base/hosts.erb index 45bf0d2..c41ef08 100644 --- a/site/profiles/templates/base/hosts.erb +++ b/site/profiles/templates/base/hosts.erb @@ -1,15 +1,10 @@ # /etc/hosts file managed by Puppet # The following lines are desirable for IPv4 capable hosts -127.0.0.1 <%= @fqdn %> <%= @hostname %> +<%= @facts['networking']['ip'] %> <%= @fqdn %> <%= @hostname %> 127.0.0.1 localhost.localdomain localhost 127.0.0.1 localhost4.localdomain4 localhost4 -# The following lines are desirable for IPv6 capable hosts -::1 <%= @fqdn %> <%= @hostname %> -::1 localhost.localdomain localhost -::1 localhost6.localdomain6 localhost6 - <% @additional_hosts.each do |host| -%> <%= host['ip'] %> <%= host['hostname'] %> <%= host['aliases'].join(' ') if host['aliases'] %> <% end -%> diff --git a/site/profiles/templates/proxmox/join_pvecluster.erb b/site/profiles/templates/proxmox/join_pvecluster.erb new file mode 100644 index 0000000..378b95d --- /dev/null +++ b/site/profiles/templates/proxmox/join_pvecluster.erb @@ -0,0 +1,11 @@ +#!/usr/bin/expect -f +set timeout -1 +set password [lindex $argv 0] +set ip [lindex $argv 1] + +spawn pvecm add $ip +expect "Please enter superuser (root) password for" +send "$password\r" +expect "The authenticity of host" +send "yes\r" +expect eof diff --git a/site/profiles/templates/proxmox/pve_facts.yaml.erb b/site/profiles/templates/proxmox/pve_facts.yaml.erb new file mode 100644 index 0000000..7b3362e --- /dev/null +++ b/site/profiles/templates/proxmox/pve_facts.yaml.erb @@ -0,0 +1,2 @@ +--- +pve_clusterinit_master: <%= @clusterinit_master %> diff --git a/site/roles/manifests/infra/proxmox/node.pp b/site/roles/manifests/infra/proxmox/node.pp new file mode 100644 index 0000000..62bc14f --- /dev/null +++ b/site/roles/manifests/infra/proxmox/node.pp @@ -0,0 +1,6 @@ +# manage the installation of a proxmox node +class roles::infra::proxmox::node { + include profiles::defaults + include profiles::base + include profiles::proxmox::init +}