feat: update settings for ceph

- add ceph package
- manage ceph directories
- update ssh principals and listening interfaces
- fix: resolve error between python3-devel and ceph
- fix: mtu issues preventing ceph syncing
- feat: manage ceph client mounts

hieradata/common.yaml

@@ -355,6 +355,7 @@ networking::route_defaults:
   netmask: 0.0.0.0
   network: default
 
+# FIXME these are for the proxmox ceph cluster
 profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8
 profiles::ceph::client::mons:
   - 10.18.15.1

hieradata/roles/infra/incus/node.eyaml

@@ -0,0 +1,2 @@
+ceph::key::media: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAX4pXMYx0Kms74s+CCT9EnuVSq3fvlcxpkezFt/VYR0cfo8K8Sm0CE0MFEQ/sdZJ+bnycjWhnkRrT0Wj2gCcqG5+KSJPVb6OOF/zkCMEjLu7xSatKakdOmnTxJEx47Lo84dGVk+YqiTT1E5XVoKlFUvZw7iIW6BgTSORo75fC2VE8MsV0GVMIusgjqMuWgyhVUGKkPGUsGAPs5Ky7uys9tlyH5s4FiZqUKvPdeKlX0HMH5XrHuKBSJg+Nju7GLNLB+/XjBsTZrY2OMC0fzczb1EQ5KL2Pl502sE8vr1VNbXqDg7vZGzWnI60Yv2t2GUxLpUjM741NP9jZ0ovGQT8I2DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDUnyCYnAG1HFiBAef1KYKsgDCtxa4fw0zb7DFzVxjpM1n5fAabbaAxIXaOxLC8u3XM3+5CYsG1dWTIDZOo+qkS9sY=]
+ceph::key::apps: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANWnFzl98B+YVXXfj5zYF5XAFbZ8iGu2VZeF53QiYxuTmRCT2/3GI4h1WB/NYcGYgmxynEsdiJus7uRO8dplsJsm+lRNffTwjnC0oDxWI82C/QuccR+YxuywqnbHLFXxZecU6/joHewt8nIu9unmF92552pqOx30ashb30L0ptR3BYBy5e9eGjjWnMrKPM4wU5NJrpW8fb9OibMWWZ1JON9tq8QITm2USCHkVOAPCfo94Dlo+2mdQWtn0GhFnRaiTNNnKT1zqSdagOUOkkY6v3yDQdmOKtR77R8bJLo/WDxUodKOt6Oi8Iuvkkvjjk6t/u05eQTPNoVQo6blV13qoDzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDWHi5X6rS0BOznTJZH1IfBgDBFQ7cIEZWzVqrEkKAb7FXdK9U+/Z2Coda3GTZ0FPJ2SzVoAV9CXNuw4808RMsS6RU=]

hieradata/roles/infra/incus/node.yaml

@@ -4,6 +4,12 @@ hiera_include:
   - frrouting
   - incus
   - zfs
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - profiles::storage::cephfsvols
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
 
 profiles::packages::include:
   bridge-utils: {}
@@ -25,15 +31,9 @@ profiles::ssh::sign::principals:
   - incus.query.consul
   - "incus.service.%{facts.country}-%{facts.region}.consul"
   - "%{hiera('networking_loopback0_ip')}"
-  - "%{hiera('networking_loopback1_ip')}"
+  - "%{facts.networking.interfaces.enp2s0.ip}"
-  - "%{hiera('networking_loopback2_ip')}"
   - "%{facts.networking.interfaces.enp3s0.ip}"
 
-profiles::accounts::root::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-profiles::accounts::sysadmin::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-
 # configure consul service
 consul::services:
   incus:
@@ -108,24 +108,24 @@ networking::interfaces:
     forwarding: true
   enp3s0:
     type: physical
-    mtu: 9000
+    mtu: 1500
     txqueuelen: 10000
     forwarding: true
   loopback0:
     type: dummy
     ipaddress: "%{hiera('networking_loopback0_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback1:
     type: dummy
     ipaddress: "%{hiera('networking_loopback1_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback2:
     type: dummy
     ipaddress: "%{hiera('networking_loopback2_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
 
 # frrouting
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
@@ -155,8 +155,7 @@ frrouting::daemons:
 ssh::server::options:
   ListenAddress:
     - "%{hiera('networking_loopback0_ip')}"
-    - "%{hiera('networking_loopback1_ip')}"
+    - "%{facts.networking.interfaces.enp2s0.ip}"
-    - "%{hiera('networking_loopback2_ip')}"
     - "%{facts.networking.interfaces.enp3s0.ip}"
 
 # zfs settings
@@ -193,6 +192,39 @@ incus::server_addr: "%{hiera('networking_loopback0_ip')}"
 profiles::accounts::sysadmin::extra_groups:
   - incus-admin
 
+# manage cephfs mounts
+profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_package: false
+profiles::ceph::client::manage_ceph_paths: false
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+profiles::ceph::client::mons:
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+profiles::ceph::client::keyrings:
+  media:
+    key: "%{hiera('ceph::key::media')}"
+  apps:
+    key: "%{hiera('ceph::key::apps')}"
+
+profiles::storage::cephfsvols::volumes:
+  cephfsvol_media:
+    mount: "/shared/media"
+    keyring: "/etc/ceph/ceph.client.media.keyring"
+    cephfs_name: "media"
+    cephfs_fs: "mediafs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[media]"
+  cephfsvol_apps:
+    mount: "/shared/apps"
+    keyring: "/etc/ceph/ceph.client.apps.keyring"
+    cephfs_name: "apps"
+    cephfs_fs: "appfs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[apps]"
+
 # sysctl recommendations
 sysctl::base::values:
   fs.aio-max-nr:

site/profiles/manifests/ceph/client.pp

@@ -3,6 +3,9 @@ class profiles::ceph::client (
   String $fsid,
   Array[Stdlib::Host] $mons,
   Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf',
+  Boolean $manage_ceph_conf = true,
+  Boolean $manage_ceph_package = true,
+  Boolean $manage_ceph_paths = true,
   String $owner = 'ceph',
   String $group = 'ceph',
   Stdlib::Filemode $mode = '0644',
@@ -13,27 +16,33 @@ class profiles::ceph::client (
   if $facts['enc_role'] != 'roles::infra::proxmox::node' {
 
     # install the ceph client package
-    package { 'ceph-common':
+    if $manage_ceph_package {
-      ensure => installed,
+      package { 'ceph-common':
+        ensure => installed,
+      }
     }
 
     # manage the ceph directory
-    file { '/etc/ceph':
+    if $manage_ceph_paths {
-      ensure  => directory,
+      file { '/etc/ceph':
-      owner   => $owner,
+        ensure  => directory,
-      group   => $group,
+        owner   => $owner,
-      mode    => $mode,
+        group   => $group,
-      require => Package['ceph-common'],
+        mode    => $mode,
+        require => Package['ceph-common'],
+      }
     }
 
     # create a basic client config
-    file { $config_file:
+    if $manage_ceph_conf {
-      ensure  => file,
+      file { $config_file:
-      owner   => $owner,
+        ensure  => file,
-      group   => $group,
+        owner   => $owner,
-      mode    => $mode,
+        group   => $group,
-      content => template('profiles/ceph/client.conf.erb'),
+        mode    => $mode,
-      require => Package['ceph-common'],
+        content => template('profiles/ceph/client.conf.erb'),
+        require => Package['ceph-common'],
+      }
     }
 
     # manage ceph keyrings

site/profiles/manifests/ceph/node.pp

@@ -0,0 +1,31 @@
+class profiles::ceph::node (
+
+){
+
+  package {'ceph':
+    ensure => 'installed',
+  }
+
+  file {'/etc/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file {'/var/log/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  # run sudo pip3 install CherryPy==18.10.0
+  # unless:
+  # [sysadmin@prodnxsr0009 ~]$ sudo pip3.9 list | grep -i cherrypy
+  # CherryPy           18.10.0
+
+
+}

site/profiles/manifests/storage/cephfsvols.pp

@@ -0,0 +1,36 @@
+# a class to manage the cephfsvol defines
+class profiles::storage::cephfsvols (
+  Hash[String, Hash] $volumes,
+) {
+
+  $volumes.each |String $title, Hash $params| {
+
+    $ensure        = pick($params['ensure'],        'mounted')
+    $owner         = pick($params['owner'],         'root')
+    $group         = pick($params['group'],         'root')
+    $mode          = pick($params['mode'],          '0755')
+    $mount         = $params['mount']
+    $mount_options = pick($params['mount_options'], ['noatime', 'nodiratime'])
+    $cephfs_mon    = pick($params['cephfs_mon'],    'ceph-mon.service.consul')
+    $cephfs_path   = pick($params['cephfs_path'],   '/')
+    $cephfs_name   = $params['cephfs_name']
+    $cephfs_fs     = $params['cephfs_fs']
+    $keyring       = $params['keyring']
+
+    profiles::storage::cephfsvol { $title:
+      ensure        => $ensure,
+      owner         => $owner,
+      group         => $group,
+      mode          => $mode,
+      mount         => $mount,
+      mount_options => $mount_options,
+      cephfs_mon    => $cephfs_mon,
+      cephfs_path   => $cephfs_path,
+      cephfs_name   => $cephfs_name,
+      cephfs_fs     => $cephfs_fs,
+      keyring       => $keyring,
+      # Optional metaparameters like `require`
+      *             => $params.filter |$k, $v| { $k in ['require', 'before', 'notify', 'subscribe'] },
+    }
+  }
+}