From 03094712d547b56ef934a0365a671df4de233e4d Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 6 May 2025 22:36:14 +1000 Subject: [PATCH] feat: deploy ceph - cleanup subnet_facts, add transit links - cleanup role::ceph::* - add openstack-ceph module - add ceph-mon profile --- Puppetfile | 2 + hieradata/roles/infra/incus/node.yaml | 15 ++++++ modules/libs/lib/facter/subnet_facts.rb | 2 +- site/profiles/manifests/ceph/mon.pp | 62 +++++++++++++++++++++++++ site/roles/manifests/ceph/mds.pp | 6 --- site/roles/manifests/ceph/mon.pp | 6 --- site/roles/manifests/ceph/osd.pp | 6 --- 7 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 site/profiles/manifests/ceph/mon.pp delete mode 100644 site/roles/manifests/ceph/mds.pp delete mode 100644 site/roles/manifests/ceph/mon.pp delete mode 100644 site/roles/manifests/ceph/osd.pp diff --git a/Puppetfile b/Puppetfile index b559a1b..17cd72d 100644 --- a/Puppetfile +++ b/Puppetfile @@ -60,6 +60,8 @@ mod 'rehan-mkdir', '2.0.0' mod 'tailoredautomation-patroni', '2.0.0' mod 'ssm-crypto_policies', '0.3.3' mod 'thias-sysctl', '1.0.8' +mod 'openstack-ceph', '7.0.0' + mod 'bind', :git => 'https://git.service.au-syd1.consul/unkinben/puppet-bind.git', diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml index 1fbe7ba..f3ae4e3 100644 --- a/hieradata/roles/infra/incus/node.yaml +++ b/hieradata/roles/infra/incus/node.yaml @@ -4,6 +4,7 @@ hiera_include: - frrouting - incus - zfs + - profiles::ceph::mon profiles::packages::include: bridge-utils: {} @@ -51,6 +52,20 @@ profiles::consul::client::node_rules: # additional repos profiles::yum::global::repos: + ceph: + name: ceph + descr: ceph repository + target: /etc/yum.repos.d/ceph.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture} + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent + ceph-noarch: + name: ceph-noarch + descr: ceph-noarch repository + target: /etc/yum.repos.d/ceph-noarch.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent frr-extras: name: frr-extras descr: frr-extras repository diff --git a/modules/libs/lib/facter/subnet_facts.rb b/modules/libs/lib/facter/subnet_facts.rb index 6bc2886..8fe67fd 100644 --- a/modules/libs/lib/facter/subnet_facts.rb +++ b/modules/libs/lib/facter/subnet_facts.rb @@ -12,7 +12,7 @@ class SubnetAttributes '198.18.17.0/24' => { environment: 'prod', region: 'drw1', country: 'au' }, '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' }, '198.18.19.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # loopbacks - '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # MPLS CORE BLOCKS + '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # transit blocks '198.18.21.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # physical network 2.5gbe '198.18.22.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph cluster '198.18.23.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph public diff --git a/site/profiles/manifests/ceph/mon.pp b/site/profiles/manifests/ceph/mon.pp new file mode 100644 index 0000000..50dc769 --- /dev/null +++ b/site/profiles/manifests/ceph/mon.pp @@ -0,0 +1,62 @@ +class profiles::ceph::mon ( + Boolean $members_lookup = true, + String $members_role = 'roles::infra::incus::node', + String $master = 'prodnxsr0009.main.unkin.net', +){ + + $admin_key = 'AQAN/iRooIyMCRAAynGdDgfEZX79YycRZcp6tw==' + $mon_key = 'AQAa/iRo37CdIhAAERhnXqDVs1BaMcVVIBc3Ew==' + $bootstrap_osd_key = 'AQAo/iRoZx0wFxAAhVVQ0BkfBpIeL6l1kdLBIw==' + $fsid = 'FBDBD9F1-9606-42D2-9C93-0E9A73BBF2C2' + + # if lookup is enabled + if $members_lookup { + + # check that the role is also set + unless !($members_role == undef) { + fail("members_role must be provided for ${title} when members_lookup is True") + } + + # if it is, find hosts, sort them so they dont cause changes every run + $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.hostname')) + $servers_ip = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.interfaces.loopback2.ip')) + } + + if length($servers_array) >= 3 { + + $servers_hostname_string = join($servers_array, ',') + $servers_ip_string = join($servers_ip, ',') + + class { 'ceph': + fsid => $fsid, + mon_initial_members => $servers_hostname_string, + mon_host => $servers_ip_string, + cluster_network => '198.18.22.0/24', + public_network => '198.18.23.0/24', + public_addr => "%{hiera('networking_loopback2_ip')}", + } + + ceph::mon { $facts['networking']['hostname']: + key => $mon_key, + public_addr => $facts['networking']['interfaces']['loopback2']['ip'], + } + + Ceph::Key { + inject => true, + inject_as_id => 'mon.', + inject_keyring => "/var/lib/ceph/mon/ceph-${facts['networking']['hostname']}/keyring", + } + + ceph::key { 'client.admin': + secret => $admin_key, + cap_mon => 'allow *', + cap_osd => 'allow *', + cap_mds => 'allow', + } + + ceph::key { 'client.bootstrap-osd': + secret => $bootstrap_osd_key, + cap_mon => 'allow profile bootstrap-osd', + } + } +} diff --git a/site/roles/manifests/ceph/mds.pp b/site/roles/manifests/ceph/mds.pp deleted file mode 100644 index a7a6a2e..0000000 --- a/site/roles/manifests/ceph/mds.pp +++ /dev/null @@ -1,6 +0,0 @@ -# a role to deploy the ceph mds -# work in progress -class roles::ceph::mds { - include profiles::defaults - include profiles::base -} diff --git a/site/roles/manifests/ceph/mon.pp b/site/roles/manifests/ceph/mon.pp deleted file mode 100644 index b1fe65a..0000000 --- a/site/roles/manifests/ceph/mon.pp +++ /dev/null @@ -1,6 +0,0 @@ -# a role to deploy the ceph mon -# work in progress -class roles::ceph::mon { - include profiles::defaults - include profiles::base -} diff --git a/site/roles/manifests/ceph/osd.pp b/site/roles/manifests/ceph/osd.pp deleted file mode 100644 index 047718a..0000000 --- a/site/roles/manifests/ceph/osd.pp +++ /dev/null @@ -1,6 +0,0 @@ -# a role to deploy the ceph osd -# work in progress -class roles::ceph::osd { - include profiles::defaults - include profiles::base -}