From 2d125487c3059a8c6bc34def76899321c9fdc773 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 8 Aug 2026 19:56:49 +1000 Subject: [PATCH 1/3] ceph: manage /etc/ceph/ceph.conf on osd and mon/mgr/mds hosts Bring the hand-maintained /etc/ceph/ceph.conf under Puppet on the prodnxsr ceph cluster (fsid de96a98f). The file is identified per host by role: k8s roles include profiles::ceph::osd only and get a [global] section; the incus node role also includes profiles::ceph::mds and additionally gets the [mds] + [mds.*] sections. - Add cluster topology as a single source of truth in common.yaml: cluster_public_ips (all 13 ceph host /32s), mon_initial_members, and the mds_instances map (two mds daemons per mon/mgr/mds host). - Render /etc/ceph/ceph.conf from that topology in the reworked client.conf.erb, preserving the live two-space indent and key order. - Gate the [mds] sections on render_mds_config, set true only in the incus node role hiera (the role that includes profiles::ceph::mds). - Drop the hard Package[ceph-common] dependency when the class does not manage the package (cephadm/profiles::packages deliver it on the k8s and incus hosts). - Enable manage_ceph_conf on the k8s and incus node roles. public_network is normalized to all 13 ceph host /32s on every host; this rewrites it on the mon/mgr/mds hosts (adding .1-.8) as the one intended content change. RGW hosts keep their profiles::ceph::conf variant and are untouched. Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT --- hieradata/common.yaml | 33 +++++++++++++++++++ hieradata/roles/infra/incus/node.yaml | 4 ++- hieradata/roles/infra/k8s.yaml | 2 +- hieradata/roles/infra/k8s/node.yaml | 2 +- site/profiles/manifests/ceph/client.pp | 34 ++++++++++++++++++-- site/profiles/templates/ceph/client.conf.erb | 27 ++++++++++++++-- 6 files changed, 94 insertions(+), 8 deletions(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 0524318..4d94c7c 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -409,6 +409,39 @@ profiles::ceph::client::mons: - 10.18.15.2 - 10.18.15.3 +# de96a98f cluster topology (prodnxsr0001-0013) - single source of truth for +# /etc/ceph/ceph.conf rendered by profiles::ceph::client on the k8s (osd) and +# incus (mon/mgr/mds) roles. fsid/mons are overridden per-role in the role hiera. +# public_network is the /32 of every ceph host's ceph-public loopback. +profiles::ceph::client::cluster_public_ips: + - 198.18.23.1 + - 198.18.23.2 + - 198.18.23.3 + - 198.18.23.4 + - 198.18.23.5 + - 198.18.23.6 + - 198.18.23.7 + - 198.18.23.8 + - 198.18.23.9 + - 198.18.23.10 + - 198.18.23.11 + - 198.18.23.12 + - 198.18.23.13 +profiles::ceph::client::mon_initial_members: + - prodnxsr0009 + - prodnxsr0010 + - prodnxsr0011 + - prodnxsr0012 + - prodnxsr0013 +# two mds daemon instances per mon/mgr/mds host (rendered only where +# render_mds_config is true, i.e. the incus node role). +profiles::ceph::client::mds_instances: + prodnxsr0009: 2 + prodnxsr0010: 2 + prodnxsr0011: 2 + prodnxsr0012: 2 + prodnxsr0013: 2 + profiles::ceph::conf::config: global: auth_client_required: 'cephx' diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml index cc68287..1cb9fbb 100644 --- a/hieradata/roles/infra/incus/node.yaml +++ b/hieradata/roles/infra/incus/node.yaml @@ -203,7 +203,9 @@ profiles::accounts::sysadmin::extra_groups: - incus-admin # manage cephfs mounts -profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_conf: true +# mon/mgr/mds host: render the [mds] + [mds.*] sections +profiles::ceph::client::render_mds_config: true profiles::ceph::client::manage_ceph_package: false profiles::ceph::client::manage_ceph_paths: false profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' diff --git a/hieradata/roles/infra/k8s.yaml b/hieradata/roles/infra/k8s.yaml index b91b86a..fa9131a 100644 --- a/hieradata/roles/infra/k8s.yaml +++ b/hieradata/roles/infra/k8s.yaml @@ -92,7 +92,7 @@ profiles::packages::include: profiles::selinux::setenforce::mode: disabled -profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_conf: true profiles::ceph::client::manage_ceph_package: false profiles::ceph::client::manage_ceph_paths: false profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' diff --git a/hieradata/roles/infra/k8s/node.yaml b/hieradata/roles/infra/k8s/node.yaml index 6dfb9ae..9aca0a0 100644 --- a/hieradata/roles/infra/k8s/node.yaml +++ b/hieradata/roles/infra/k8s/node.yaml @@ -33,7 +33,7 @@ profiles::packages::include: profiles::selinux::setenforce::mode: disabled -profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_conf: true profiles::ceph::client::manage_ceph_package: false profiles::ceph::client::manage_ceph_paths: false profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' diff --git a/site/profiles/manifests/ceph/client.pp b/site/profiles/manifests/ceph/client.pp index db7187c..4561d6a 100644 --- a/site/profiles/manifests/ceph/client.pp +++ b/site/profiles/manifests/ceph/client.pp @@ -2,6 +2,9 @@ class profiles::ceph::client ( String $fsid, Array[Stdlib::Host] $mons, + # cluster topology (single source of truth: hieradata/common.yaml) + Array[Stdlib::Host] $cluster_public_ips, + Array[Stdlib::Host] $mon_initial_members, Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf', Boolean $manage_ceph_conf = true, Boolean $manage_ceph_package = true, @@ -10,6 +13,24 @@ class profiles::ceph::client ( String $group = 'ceph', Stdlib::Filemode $mode = '0644', Hash $keyrings = {}, + # [global] tunables (defaults match the live hand-maintained ceph.conf) + String $auth_client_required = 'cephx', + String $auth_cluster_required = 'cephx', + String $auth_service_required = 'cephx', + Boolean $mon_allow_pool_delete = true, + Boolean $ms_bind_ipv4 = true, + Boolean $ms_bind_ipv6 = false, + Integer $osd_crush_chooseleaf_type = 1, + Integer $osd_pool_default_min_size = 2, + Integer $osd_pool_default_size = 3, + Integer $osd_pool_default_pg_num = 128, + # mds config sections; only rendered on mon/mgr/mds hosts (render_mds_config) + Boolean $render_mds_config = false, + Hash[String, Integer] $mds_instances = {}, + Hash $mds_common = { + 'keyring' => '/var/lib/ceph/mds/ceph-$id/keyring', + 'mds_standby_replay' => true, + }, ) { # dont run this on proxmox nodes @@ -22,6 +43,13 @@ class profiles::ceph::client ( } } + # only depend on the package when this class manages it; on the ceph + # hosts the package is delivered by cephadm / profiles::packages instead. + $config_require = $manage_ceph_package ? { + true => Package['ceph-common'], + default => undef, + } + # manage the ceph directory if $manage_ceph_paths { file { '/etc/ceph': @@ -29,11 +57,11 @@ class profiles::ceph::client ( owner => $owner, group => $group, mode => $mode, - require => Package['ceph-common'], + require => $config_require, } } - # create a basic client config + # render /etc/ceph/ceph.conf from cluster topology in hiera if $manage_ceph_conf { file { $config_file: ensure => file, @@ -41,7 +69,7 @@ class profiles::ceph::client ( group => $group, mode => $mode, content => template('profiles/ceph/client.conf.erb'), - require => Package['ceph-common'], + require => $config_require, } } diff --git a/site/profiles/templates/ceph/client.conf.erb b/site/profiles/templates/ceph/client.conf.erb index af4c678..ec851b2 100644 --- a/site/profiles/templates/ceph/client.conf.erb +++ b/site/profiles/templates/ceph/client.conf.erb @@ -1,3 +1,26 @@ [global] - fsid = <%= @fsid %> - mon_host = <%= @mons.join(' ') %> + auth_client_required = <%= @auth_client_required %> + auth_cluster_required = <%= @auth_cluster_required %> + auth_service_required = <%= @auth_service_required %> + fsid = <%= @fsid %> + mon_allow_pool_delete = <%= @mon_allow_pool_delete %> + mon_initial_members = <%= @mon_initial_members.join(',') %> + mon_host = <%= @mons.join(',') %> + ms_bind_ipv4 = <%= @ms_bind_ipv4 %> + ms_bind_ipv6 = <%= @ms_bind_ipv6 %> + osd_crush_chooseleaf_type = <%= @osd_crush_chooseleaf_type %> + osd_pool_default_min_size = <%= @osd_pool_default_min_size %> + osd_pool_default_size = <%= @osd_pool_default_size %> + osd_pool_default_pg_num = <%= @osd_pool_default_pg_num %> + public_network = <%= @cluster_public_ips.map { |ip| "#{ip}/32" }.join(',') %> +<% if @render_mds_config -%> +[mds] + keyring = <%= @mds_common['keyring'] %> + mds_standby_replay = <%= @mds_common['mds_standby_replay'] %> +<% @mds_instances.sort.each do |host, count| -%> +<% (1..count).each do |instance| -%> +[mds.<%= host %>-<%= instance %>] + host = <%= host %> +<% end -%> +<% end -%> +<% end -%> -- 2.47.3 From 5b04aa341d784f0f5594072ac25f1246daec0eee Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 8 Aug 2026 20:04:04 +1000 Subject: [PATCH 2/3] ceph: match live blank-line separators between mds sections Diffing the render against the live /etc/ceph/ceph.conf on prodnxsr0009 showed the hand-maintained file separates the mds sections with blank lines: one before [mds] and one before each [mds.X-i] block (no trailing blank line after the last). Emit those separators so a mon/mgr/mds host renders byte-identical to live except for the intended public_network normalization. Verified: osd host stays byte-identical, mon host differs only on the public_network line. Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT --- site/profiles/templates/ceph/client.conf.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/site/profiles/templates/ceph/client.conf.erb b/site/profiles/templates/ceph/client.conf.erb index ec851b2..470ec9e 100644 --- a/site/profiles/templates/ceph/client.conf.erb +++ b/site/profiles/templates/ceph/client.conf.erb @@ -14,11 +14,13 @@ osd_pool_default_pg_num = <%= @osd_pool_default_pg_num %> public_network = <%= @cluster_public_ips.map { |ip| "#{ip}/32" }.join(',') %> <% if @render_mds_config -%> + [mds] keyring = <%= @mds_common['keyring'] %> mds_standby_replay = <%= @mds_common['mds_standby_replay'] %> <% @mds_instances.sort.each do |host, count| -%> <% (1..count).each do |instance| -%> + [mds.<%= host %>-<%= instance %>] host = <%= host %> <% end -%> -- 2.47.3 From fa3b97058c10c53b619a9b25775b8f4c5edad80f Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 8 Aug 2026 22:25:53 +1000 Subject: [PATCH 3/3] ceph: include prodnxsr0014-0019 in public_network Why: prodnxsr0014-0019 were racked as roles::base but have since joined the de96a98f ceph cluster as OSD hosts (live enc_role is now roles::infra::k8s::compute, is_ceph_osd true). Their ceph-public /32s must appear in public_network on every cluster member, and they must receive the osd-only ceph.conf. How: - Expand profiles::ceph::client::cluster_public_ips from .1-.13 to .1-.19 (add ceph-public loopbacks 198.18.23.14-19). - No role/hiera change needed for 0014-0019: they run roles::infra::k8s::compute, already covered by roles/infra/k8s.yaml (manage_ceph_conf true, render_mds_config unset/false), so they get the [global]-only variant with no mds sections. Consequence: first convergence now rewrites public_network on the existing osd hosts (0001-0008) and mon hosts (0009-0013) too, adding .14-.19; adoption is a single public_network-line change on every cluster host rather than a no-op on osd hosts. Verified against the live files: osd (0008) and mon (0009) each differ by exactly the public_network line, mds formatting on 0009 unchanged. --- hieradata/common.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 4d94c7c..a0939e5 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -409,7 +409,7 @@ profiles::ceph::client::mons: - 10.18.15.2 - 10.18.15.3 -# de96a98f cluster topology (prodnxsr0001-0013) - single source of truth for +# de96a98f cluster topology (prodnxsr0001-0019) - single source of truth for # /etc/ceph/ceph.conf rendered by profiles::ceph::client on the k8s (osd) and # incus (mon/mgr/mds) roles. fsid/mons are overridden per-role in the role hiera. # public_network is the /32 of every ceph host's ceph-public loopback. @@ -427,6 +427,12 @@ profiles::ceph::client::cluster_public_ips: - 198.18.23.11 - 198.18.23.12 - 198.18.23.13 + - 198.18.23.14 + - 198.18.23.15 + - 198.18.23.16 + - 198.18.23.17 + - 198.18.23.18 + - 198.18.23.19 profiles::ceph::client::mon_initial_members: - prodnxsr0009 - prodnxsr0010 -- 2.47.3