feat: deploy ceph

- cleanup subnet_facts, add transit links
- cleanup role::ceph::*
- add openstack-ceph module
- add ceph-mon profile

Puppetfile

@@ -60,6 +60,8 @@ mod 'rehan-mkdir', '2.0.0'
 mod 'tailoredautomation-patroni', '2.0.0'
 mod 'ssm-crypto_policies', '0.3.3'
 mod 'thias-sysctl', '1.0.8'
+mod 'openstack-ceph', '7.0.0'
+
 
 mod 'bind',
   :git => 'https://git.service.au-syd1.consul/unkinben/puppet-bind.git',

hieradata/roles/infra/incus/node.yaml

@@ -4,6 +4,7 @@ hiera_include:
   - frrouting
   - incus
   - zfs
+  - profiles::ceph::mon
 
 profiles::packages::include:
   bridge-utils: {}
@@ -51,6 +52,20 @@ profiles::consul::client::node_rules:
 
 # additional repos
 profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
   frr-extras:
     name: frr-extras
     descr: frr-extras repository

modules/libs/lib/facter/subnet_facts.rb

@@ -12,7 +12,7 @@ class SubnetAttributes
     '198.18.17.0/24' => { environment: 'prod', region: 'drw1', country: 'au' },
     '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' },
     '198.18.19.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # loopbacks
-    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # MPLS CORE BLOCKS
+    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # transit blocks
     '198.18.21.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # physical network 2.5gbe
     '198.18.22.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph cluster
     '198.18.23.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph public

site/profiles/manifests/ceph/mon.pp

@@ -0,0 +1,62 @@
+class profiles::ceph::mon (
+  Boolean $members_lookup    = true,
+  String  $members_role      = 'roles::infra::incus::node',
+  String  $master            = 'prodnxsr0009.main.unkin.net',
+){
+
+  $admin_key = 'AQAN/iRooIyMCRAAynGdDgfEZX79YycRZcp6tw=='
+  $mon_key = 'AQAa/iRo37CdIhAAERhnXqDVs1BaMcVVIBc3Ew=='
+  $bootstrap_osd_key = 'AQAo/iRoZx0wFxAAhVVQ0BkfBpIeL6l1kdLBIw=='
+  $fsid = 'FBDBD9F1-9606-42D2-9C93-0E9A73BBF2C2'
+
+  # if lookup is enabled
+  if $members_lookup {
+
+    # check that the role is also set
+    unless !($members_role == undef) {
+      fail("members_role must be provided for ${title} when members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.hostname'))
+    $servers_ip = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.interfaces.loopback2.ip'))
+  }
+
+  if length($servers_array) >= 3 {
+
+    $servers_hostname_string = join($servers_array, ',')
+    $servers_ip_string = join($servers_ip, ',')
+
+    class { 'ceph':
+      fsid                => $fsid,
+      mon_initial_members => $servers_hostname_string,
+      mon_host            => $servers_ip_string,
+      cluster_network     => '198.18.22.0/24',
+      public_network      => '198.18.23.0/24',
+      public_addr         => "%{hiera('networking_loopback2_ip')}",
+    }
+
+    ceph::mon { $facts['networking']['hostname']:
+      key         => $mon_key,
+      public_addr => $facts['networking']['interfaces']['loopback2']['ip'],
+    }
+
+    Ceph::Key {
+      inject         => true,
+      inject_as_id   => 'mon.',
+      inject_keyring => "/var/lib/ceph/mon/ceph-${facts['networking']['hostname']}/keyring",
+    }
+
+    ceph::key { 'client.admin':
+      secret  => $admin_key,
+      cap_mon => 'allow *',
+      cap_osd => 'allow *',
+      cap_mds => 'allow',
+    }
+
+    ceph::key { 'client.bootstrap-osd':
+      secret  => $bootstrap_osd_key,
+      cap_mon => 'allow profile bootstrap-osd',
+    }
+  }
+}

site/roles/manifests/ceph/mds.pp

@@ -1,6 +0,0 @@
-# a role to deploy the ceph mds
-# work in progress
-class roles::ceph::mds {
-    include profiles::defaults
-    include profiles::base
-}

site/roles/manifests/ceph/mon.pp

@@ -1,6 +0,0 @@
-# a role to deploy the ceph mon
-# work in progress
-class roles::ceph::mon {
-    include profiles::defaults
-    include profiles::base
-}

site/roles/manifests/ceph/osd.pp

@@ -1,6 +0,0 @@
-# a role to deploy the ceph osd
-# work in progress
-class roles::ceph::osd {
-    include profiles::defaults
-    include profiles::base
-}