feat: update settings for ceph

- add ceph package
- manage ceph directories
- update ssh principals and listening interfaces
- fix: resolve error between python3-devel and ceph
- fix: mtu issues preventing ceph syncing
- feat: manage ceph client mounts

hieradata/roles/infra/incus/node.eyaml

@@ -0,0 +1,2 @@
+ceph::key::media: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAX4pXMYx0Kms74s+CCT9EnuVSq3fvlcxpkezFt/VYR0cfo8K8Sm0CE0MFEQ/sdZJ+bnycjWhnkRrT0Wj2gCcqG5+KSJPVb6OOF/zkCMEjLu7xSatKakdOmnTxJEx47Lo84dGVk+YqiTT1E5XVoKlFUvZw7iIW6BgTSORo75fC2VE8MsV0GVMIusgjqMuWgyhVUGKkPGUsGAPs5Ky7uys9tlyH5s4FiZqUKvPdeKlX0HMH5XrHuKBSJg+Nju7GLNLB+/XjBsTZrY2OMC0fzczb1EQ5KL2Pl502sE8vr1VNbXqDg7vZGzWnI60Yv2t2GUxLpUjM741NP9jZ0ovGQT8I2DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDUnyCYnAG1HFiBAef1KYKsgDCtxa4fw0zb7DFzVxjpM1n5fAabbaAxIXaOxLC8u3XM3+5CYsG1dWTIDZOo+qkS9sY=]
+ceph::key::apps: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANWnFzl98B+YVXXfj5zYF5XAFbZ8iGu2VZeF53QiYxuTmRCT2/3GI4h1WB/NYcGYgmxynEsdiJus7uRO8dplsJsm+lRNffTwjnC0oDxWI82C/QuccR+YxuywqnbHLFXxZecU6/joHewt8nIu9unmF92552pqOx30ashb30L0ptR3BYBy5e9eGjjWnMrKPM4wU5NJrpW8fb9OibMWWZ1JON9tq8QITm2USCHkVOAPCfo94Dlo+2mdQWtn0GhFnRaiTNNnKT1zqSdagOUOkkY6v3yDQdmOKtR77R8bJLo/WDxUodKOt6Oi8Iuvkkvjjk6t/u05eQTPNoVQo6blV13qoDzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDWHi5X6rS0BOznTJZH1IfBgDBFQ7cIEZWzVqrEkKAb7FXdK9U+/Z2Coda3GTZ0FPJ2SzVoAV9CXNuw4808RMsS6RU=]

hieradata/roles/infra/incus/node.yaml

@@ -4,6 +4,12 @@ hiera_include:
   - frrouting
   - incus
   - zfs
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - profiles::storage::cephfsvols
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
 
 profiles::packages::include:
   bridge-utils: {}
@@ -25,15 +31,9 @@ profiles::ssh::sign::principals:
   - incus.query.consul
   - "incus.service.%{facts.country}-%{facts.region}.consul"
   - "%{hiera('networking_loopback0_ip')}"
-  - "%{hiera('networking_loopback1_ip')}"
-  - "%{hiera('networking_loopback2_ip')}"
+  - "%{facts.networking.interfaces.enp2s0.ip}"
   - "%{facts.networking.interfaces.enp3s0.ip}"
 
-profiles::accounts::root::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-profiles::accounts::sysadmin::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-
 # configure consul service
 consul::services:
   incus:
@@ -108,24 +108,24 @@ networking::interfaces:
     forwarding: true
   enp3s0:
     type: physical
-    mtu: 9000
+    mtu: 1500
     txqueuelen: 10000
     forwarding: true
   loopback0:
     type: dummy
     ipaddress: "%{hiera('networking_loopback0_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback1:
     type: dummy
     ipaddress: "%{hiera('networking_loopback1_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback2:
     type: dummy
     ipaddress: "%{hiera('networking_loopback2_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
 
 # frrouting
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
@@ -155,8 +155,7 @@ frrouting::daemons:
 ssh::server::options:
   ListenAddress:
     - "%{hiera('networking_loopback0_ip')}"
-    - "%{hiera('networking_loopback1_ip')}"
-    - "%{hiera('networking_loopback2_ip')}"
+    - "%{facts.networking.interfaces.enp2s0.ip}"
     - "%{facts.networking.interfaces.enp3s0.ip}"
 
 # zfs settings
@@ -193,6 +192,39 @@ incus::server_addr: "%{hiera('networking_loopback0_ip')}"
 profiles::accounts::sysadmin::extra_groups:
   - incus-admin
 
+# manage cephfs mounts
+profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_package: false
+profiles::ceph::client::manage_ceph_paths: false
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+profiles::ceph::client::mons:
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+profiles::ceph::client::keyrings:
+  media:
+    key: "%{hiera('ceph::key::media')}"
+  apps:
+    key: "%{hiera('ceph::key::apps')}"
+
+profiles::storage::cephfsvols::volumes:
+  cephfsvol_media:
+    mount: "/shared/media"
+    keyring: "/etc/ceph/ceph.client.media.keyring"
+    cephfs_name: "media"
+    cephfs_fs: "mediafs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[media]"
+  cephfsvol_apps:
+    mount: "/shared/apps"
+    keyring: "/etc/ceph/ceph.client.apps.keyring"
+    cephfs_name: "apps"
+    cephfs_fs: "appfs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[apps]"
+
 # sysctl recommendations
 sysctl::base::values:
   fs.aio-max-nr: