From 3cfafbac44f81b44766443f97b96f08efb2405fb Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 19 Jul 2025 20:30:46 +1000 Subject: [PATCH] feat: enable ceph on k8s nodes (#362) - enable enough ceph/frr to join to cephfs - notify sshd when restarting the network - update ssh principals to include all ssh interfaces Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/362 --- .../nodes/prodnxsr0001.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0002.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0003.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0004.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0005.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0006.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0007.main.unkin.net.yaml | 13 ++++++++++ .../nodes/prodnxsr0008.main.unkin.net.yaml | 13 ++++++++++ hieradata/roles/infra/k8s/node.yaml | 24 +++++++++++++------ modules/networking/manifests/init.pp | 1 + 10 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 hieradata/nodes/prodnxsr0001.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0002.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0003.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0004.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0005.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0006.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0007.main.unkin.net.yaml create mode 100644 hieradata/nodes/prodnxsr0008.main.unkin.net.yaml diff --git a/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml new file mode 100644 index 0000000..248a2d3 --- /dev/null +++ b/hieradata/nodes/prodnxsr0001.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.1 # management loopback +networking_loopback1_ip: 198.18.22.1 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.1 # ceph-public loopback +networking_1000_ip: 198.18.15.1 # 1gbe network +networking_2500_ip: 198.18.21.1 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:75:c3:60 + "%{hiera('networking_2500_iface')}": + mac: 00:ac:d0:00:00:50 diff --git a/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml new file mode 100644 index 0000000..6d7f8e1 --- /dev/null +++ b/hieradata/nodes/prodnxsr0002.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.2 # management loopback +networking_loopback1_ip: 198.18.22.2 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.2 # ceph-public loopback +networking_1000_ip: 198.18.15.2 # 1gbe network +networking_2500_ip: 198.18.21.2 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:74:b6:08 + "%{hiera('networking_2500_iface')}": + mac: 00:e0:4c:68:08:43 diff --git a/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml new file mode 100644 index 0000000..66fde08 --- /dev/null +++ b/hieradata/nodes/prodnxsr0003.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.3 # management loopback +networking_loopback1_ip: 198.18.22.3 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.3 # ceph-public loopback +networking_1000_ip: 198.18.15.3 # 1gbe network +networking_2500_ip: 198.18.21.3 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: b8:85:84:a3:25:c5 + "%{hiera('networking_2500_iface')}": + mac: 00:e0:4c:68:07:82 diff --git a/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml new file mode 100644 index 0000000..3aec489 --- /dev/null +++ b/hieradata/nodes/prodnxsr0004.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.4 # management loopback +networking_loopback1_ip: 198.18.22.4 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.4 # ceph-public loopback +networking_1000_ip: 198.18.15.4 # 1gbe network +networking_2500_ip: 198.18.21.4 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:75:d5:00 + "%{hiera('networking_2500_iface')}": + mac: 00:ac:d0:00:00:43 diff --git a/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml new file mode 100644 index 0000000..e2fbb14 --- /dev/null +++ b/hieradata/nodes/prodnxsr0005.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.5 # management loopback +networking_loopback1_ip: 198.18.22.5 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.5 # ceph-public loopback +networking_1000_ip: 198.18.15.5 # 1gbe network +networking_2500_ip: 198.18.21.5 # 2.5gbe network +networking_1000_iface: enp1s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: 54:bf:64:a0:08:64 + "%{hiera('networking_2500_iface')}": + mac: 00:e0:4c:68:07:79 diff --git a/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml new file mode 100644 index 0000000..66c58ab --- /dev/null +++ b/hieradata/nodes/prodnxsr0006.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.6 # management loopback +networking_loopback1_ip: 198.18.22.6 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.6 # ceph-public loopback +networking_1000_ip: 198.18.15.6 # 1gbe network +networking_2500_ip: 198.18.21.6 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:75:10:8d + "%{hiera('networking_2500_iface')}": + mac: 00:ac:d0:00:00:53 diff --git a/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml new file mode 100644 index 0000000..8b9d21a --- /dev/null +++ b/hieradata/nodes/prodnxsr0007.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.7 # management loopback +networking_loopback1_ip: 198.18.22.7 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.7 # ceph-public loopback +networking_1000_ip: 198.18.15.7 # 1gbe network +networking_2500_ip: 198.18.21.7 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:74:b4:27 + "%{hiera('networking_2500_iface')}": + mac: 00:ac:d0:00:00:5b diff --git a/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml new file mode 100644 index 0000000..80c7e23 --- /dev/null +++ b/hieradata/nodes/prodnxsr0008.main.unkin.net.yaml @@ -0,0 +1,13 @@ +--- +networking_loopback0_ip: 198.18.19.8 # management loopback +networking_loopback1_ip: 198.18.22.8 # ceph-cluster loopback +networking_loopback2_ip: 198.18.23.8 # ceph-public loopback +networking_1000_ip: 198.18.15.8 # 1gbe network +networking_2500_ip: 198.18.21.8 # 2.5gbe network +networking_1000_iface: enp2s0 +networking_2500_iface: enp3s0 +networking::interfaces: + "%{hiera('networking_1000_iface')}": + mac: d8:9e:f3:75:06:18 + "%{hiera('networking_2500_iface')}": + mac: 00:e0:4c:68:08:4b diff --git a/hieradata/roles/infra/k8s/node.yaml b/hieradata/roles/infra/k8s/node.yaml index b66f40a..c4cca0b 100644 --- a/hieradata/roles/infra/k8s/node.yaml +++ b/hieradata/roles/infra/k8s/node.yaml @@ -1,6 +1,6 @@ --- hiera_include: - - profiles::selinux::frr + - profiles::selinux::setenforce - frrouting - profiles::ceph::node - profiles::ceph::client @@ -12,6 +12,8 @@ profiles::packages::include: bridge-utils: {} cephadm: {} +profiles::selinux::setenforce::mode: disabled + profiles::ceph::client::manage_ceph_conf: false profiles::ceph::client::manage_ceph_package: false profiles::ceph::client::manage_ceph_paths: false @@ -60,12 +62,15 @@ profiles::dns::base::primary_interface: loopback0 systemd::manage_networkd: true systemd::manage_all_network_files: true networking::interfaces: - enp2s0: + "%{hiera('networking_1000_iface')}": type: physical + ipaddress: "%{hiera('networking_1000_ip')}" + gateway: 198.18.15.254 txqueuelen: 10000 forwarding: true - enp3s0: + "%{hiera('networking_2500_iface')}": type: physical + ipaddress: "%{hiera('networking_2500_ip')}" mtu: 1500 txqueuelen: 10000 forwarding: true @@ -90,9 +95,9 @@ frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}" frrouting::ospfd_redistribute: - connected frrouting::ospfd_interfaces: - enp2s0: + "%{hiera('networking_1000_iface')}": area: 0.0.0.0 - enp3s0: + "%{hiera('networking_2500_iface')}": area: 0.0.0.0 loopback0: area: 0.0.0.0 @@ -107,5 +112,10 @@ frrouting::daemons: ssh::server::options: ListenAddress: - "%{hiera('networking_loopback0_ip')}" - - "%{facts.networking.interfaces.enp2s0.ip}" - - "%{facts.networking.interfaces.enp3s0.ip}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" + +profiles::ssh::sign::principals: + - "%{hiera('networking_loopback0_ip')}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" diff --git a/modules/networking/manifests/init.pp b/modules/networking/manifests/init.pp index dd2b1f5..0546fcb 100644 --- a/modules/networking/manifests/init.pp +++ b/modules/networking/manifests/init.pp @@ -71,6 +71,7 @@ class networking ( exec { 'networking_reload_network': command => $restart_command, refreshonly => true, + notify => Service['sshd'], } # prevent DNS from being overwritten by networkmanager