From de9079e43cff69e37e6e1e28cf8e78fabd2911ba Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 17 May 2025 20:52:42 +1000 Subject: [PATCH] feat: update settings for ceph - add ceph package - manage ceph directories - update ssh principals and listening interfaces - fix: resolve error between python3-devel and ceph - fix: mtu issues preventing ceph syncing - feat: manage ceph client mounts --- hieradata/common.yaml | 1 + hieradata/roles/infra/incus/node.eyaml | 2 + hieradata/roles/infra/incus/node.yaml | 58 ++++++++++++++----- site/profiles/manifests/ceph/client.pp | 39 ++++++++----- site/profiles/manifests/ceph/node.pp | 31 ++++++++++ site/profiles/manifests/storage/cephfsvols.pp | 36 ++++++++++++ 6 files changed, 139 insertions(+), 28 deletions(-) create mode 100644 hieradata/roles/infra/incus/node.eyaml create mode 100644 site/profiles/manifests/ceph/node.pp create mode 100644 site/profiles/manifests/storage/cephfsvols.pp diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 9e0f02e..48590eb 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -355,6 +355,7 @@ networking::route_defaults: netmask: 0.0.0.0 network: default +# FIXME these are for the proxmox ceph cluster profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8 profiles::ceph::client::mons: - 10.18.15.1 diff --git a/hieradata/roles/infra/incus/node.eyaml b/hieradata/roles/infra/incus/node.eyaml new file mode 100644 index 0000000..c85b8f6 --- /dev/null +++ b/hieradata/roles/infra/incus/node.eyaml @@ -0,0 +1,2 @@ +ceph::key::media: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAX4pXMYx0Kms74s+CCT9EnuVSq3fvlcxpkezFt/VYR0cfo8K8Sm0CE0MFEQ/sdZJ+bnycjWhnkRrT0Wj2gCcqG5+KSJPVb6OOF/zkCMEjLu7xSatKakdOmnTxJEx47Lo84dGVk+YqiTT1E5XVoKlFUvZw7iIW6BgTSORo75fC2VE8MsV0GVMIusgjqMuWgyhVUGKkPGUsGAPs5Ky7uys9tlyH5s4FiZqUKvPdeKlX0HMH5XrHuKBSJg+Nju7GLNLB+/XjBsTZrY2OMC0fzczb1EQ5KL2Pl502sE8vr1VNbXqDg7vZGzWnI60Yv2t2GUxLpUjM741NP9jZ0ovGQT8I2DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDUnyCYnAG1HFiBAef1KYKsgDCtxa4fw0zb7DFzVxjpM1n5fAabbaAxIXaOxLC8u3XM3+5CYsG1dWTIDZOo+qkS9sY=] +ceph::key::apps: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANWnFzl98B+YVXXfj5zYF5XAFbZ8iGu2VZeF53QiYxuTmRCT2/3GI4h1WB/NYcGYgmxynEsdiJus7uRO8dplsJsm+lRNffTwjnC0oDxWI82C/QuccR+YxuywqnbHLFXxZecU6/joHewt8nIu9unmF92552pqOx30ashb30L0ptR3BYBy5e9eGjjWnMrKPM4wU5NJrpW8fb9OibMWWZ1JON9tq8QITm2USCHkVOAPCfo94Dlo+2mdQWtn0GhFnRaiTNNnKT1zqSdagOUOkkY6v3yDQdmOKtR77R8bJLo/WDxUodKOt6Oi8Iuvkkvjjk6t/u05eQTPNoVQo6blV13qoDzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDWHi5X6rS0BOznTJZH1IfBgDBFQ7cIEZWzVqrEkKAb7FXdK9U+/Z2Coda3GTZ0FPJ2SzVoAV9CXNuw4808RMsS6RU=] diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml index 72d7155..cd07ebc 100644 --- a/hieradata/roles/infra/incus/node.yaml +++ b/hieradata/roles/infra/incus/node.yaml @@ -4,6 +4,12 @@ hiera_include: - frrouting - incus - zfs + - profiles::ceph::node + - profiles::ceph::client + - profiles::storage::cephfsvols + +# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package +python::manage_dev_package: false profiles::packages::include: bridge-utils: {} @@ -25,15 +31,9 @@ profiles::ssh::sign::principals: - incus.query.consul - "incus.service.%{facts.country}-%{facts.region}.consul" - "%{hiera('networking_loopback0_ip')}" - - "%{hiera('networking_loopback1_ip')}" - - "%{hiera('networking_loopback2_ip')}" + - "%{facts.networking.interfaces.enp2s0.ip}" - "%{facts.networking.interfaces.enp3s0.ip}" -profiles::accounts::root::sshkeys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d -profiles::accounts::sysadmin::sshkeys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d - # configure consul service consul::services: incus: @@ -108,24 +108,24 @@ networking::interfaces: forwarding: true enp3s0: type: physical - mtu: 9000 + mtu: 1500 txqueuelen: 10000 forwarding: true loopback0: type: dummy ipaddress: "%{hiera('networking_loopback0_ip')}" netmask: 255.255.255.255 - mtu: 9000 + mtu: 1500 loopback1: type: dummy ipaddress: "%{hiera('networking_loopback1_ip')}" netmask: 255.255.255.255 - mtu: 9000 + mtu: 1500 loopback2: type: dummy ipaddress: "%{hiera('networking_loopback2_ip')}" netmask: 255.255.255.255 - mtu: 9000 + mtu: 1500 # frrouting frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}" @@ -155,8 +155,7 @@ frrouting::daemons: ssh::server::options: ListenAddress: - "%{hiera('networking_loopback0_ip')}" - - "%{hiera('networking_loopback1_ip')}" - - "%{hiera('networking_loopback2_ip')}" + - "%{facts.networking.interfaces.enp2s0.ip}" - "%{facts.networking.interfaces.enp3s0.ip}" # zfs settings @@ -193,6 +192,39 @@ incus::server_addr: "%{hiera('networking_loopback0_ip')}" profiles::accounts::sysadmin::extra_groups: - incus-admin +# manage cephfs mounts +profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_package: false +profiles::ceph::client::manage_ceph_paths: false +profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' +profiles::ceph::client::mons: + - 198.18.23.9 + - 198.18.23.10 + - 198.18.23.11 + - 198.18.23.12 + - 198.18.23.13 +profiles::ceph::client::keyrings: + media: + key: "%{hiera('ceph::key::media')}" + apps: + key: "%{hiera('ceph::key::apps')}" + +profiles::storage::cephfsvols::volumes: + cephfsvol_media: + mount: "/shared/media" + keyring: "/etc/ceph/ceph.client.media.keyring" + cephfs_name: "media" + cephfs_fs: "mediafs" + cephfs_mon: "%{alias('profiles::ceph::client::mons')}" + require: "Profiles::Ceph::Keyring[media]" + cephfsvol_apps: + mount: "/shared/apps" + keyring: "/etc/ceph/ceph.client.apps.keyring" + cephfs_name: "apps" + cephfs_fs: "appfs" + cephfs_mon: "%{alias('profiles::ceph::client::mons')}" + require: "Profiles::Ceph::Keyring[apps]" + # sysctl recommendations sysctl::base::values: fs.aio-max-nr: diff --git a/site/profiles/manifests/ceph/client.pp b/site/profiles/manifests/ceph/client.pp index 1735a19..db7187c 100644 --- a/site/profiles/manifests/ceph/client.pp +++ b/site/profiles/manifests/ceph/client.pp @@ -3,6 +3,9 @@ class profiles::ceph::client ( String $fsid, Array[Stdlib::Host] $mons, Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf', + Boolean $manage_ceph_conf = true, + Boolean $manage_ceph_package = true, + Boolean $manage_ceph_paths = true, String $owner = 'ceph', String $group = 'ceph', Stdlib::Filemode $mode = '0644', @@ -13,27 +16,33 @@ class profiles::ceph::client ( if $facts['enc_role'] != 'roles::infra::proxmox::node' { # install the ceph client package - package { 'ceph-common': - ensure => installed, + if $manage_ceph_package { + package { 'ceph-common': + ensure => installed, + } } # manage the ceph directory - file { '/etc/ceph': - ensure => directory, - owner => $owner, - group => $group, - mode => $mode, - require => Package['ceph-common'], + if $manage_ceph_paths { + file { '/etc/ceph': + ensure => directory, + owner => $owner, + group => $group, + mode => $mode, + require => Package['ceph-common'], + } } # create a basic client config - file { $config_file: - ensure => file, - owner => $owner, - group => $group, - mode => $mode, - content => template('profiles/ceph/client.conf.erb'), - require => Package['ceph-common'], + if $manage_ceph_conf { + file { $config_file: + ensure => file, + owner => $owner, + group => $group, + mode => $mode, + content => template('profiles/ceph/client.conf.erb'), + require => Package['ceph-common'], + } } # manage ceph keyrings diff --git a/site/profiles/manifests/ceph/node.pp b/site/profiles/manifests/ceph/node.pp new file mode 100644 index 0000000..df10456 --- /dev/null +++ b/site/profiles/manifests/ceph/node.pp @@ -0,0 +1,31 @@ +class profiles::ceph::node ( + +){ + + package {'ceph': + ensure => 'installed', + } + + file {'/etc/ceph': + ensure => directory, + owner => 'ceph', + group => 'ceph', + mode => '0755', + require => Package['ceph'], + } + + file {'/var/log/ceph': + ensure => directory, + owner => 'ceph', + group => 'ceph', + mode => '0755', + require => Package['ceph'], + } + + # run sudo pip3 install CherryPy==18.10.0 + # unless: + # [sysadmin@prodnxsr0009 ~]$ sudo pip3.9 list | grep -i cherrypy + # CherryPy 18.10.0 + + +} diff --git a/site/profiles/manifests/storage/cephfsvols.pp b/site/profiles/manifests/storage/cephfsvols.pp new file mode 100644 index 0000000..eb48995 --- /dev/null +++ b/site/profiles/manifests/storage/cephfsvols.pp @@ -0,0 +1,36 @@ +# a class to manage the cephfsvol defines +class profiles::storage::cephfsvols ( + Hash[String, Hash] $volumes, +) { + + $volumes.each |String $title, Hash $params| { + + $ensure = pick($params['ensure'], 'mounted') + $owner = pick($params['owner'], 'root') + $group = pick($params['group'], 'root') + $mode = pick($params['mode'], '0755') + $mount = $params['mount'] + $mount_options = pick($params['mount_options'], ['noatime', 'nodiratime']) + $cephfs_mon = pick($params['cephfs_mon'], 'ceph-mon.service.consul') + $cephfs_path = pick($params['cephfs_path'], '/') + $cephfs_name = $params['cephfs_name'] + $cephfs_fs = $params['cephfs_fs'] + $keyring = $params['keyring'] + + profiles::storage::cephfsvol { $title: + ensure => $ensure, + owner => $owner, + group => $group, + mode => $mode, + mount => $mount, + mount_options => $mount_options, + cephfs_mon => $cephfs_mon, + cephfs_path => $cephfs_path, + cephfs_name => $cephfs_name, + cephfs_fs => $cephfs_fs, + keyring => $keyring, + # Optional metaparameters like `require` + * => $params.filter |$k, $v| { $k in ['require', 'before', 'notify', 'subscribe'] }, + } + } +} -- 2.47.3