unkin/puppet-prod
3
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: adding rke2
Build / precommit (pull_request) Successful in 5m6s
Details

Browse Source 
- manage rke2 repos
- add rke2 module (init, params, install, config, service)
- split roles::infra::k8s::node -> control/compute roles
- moved common k8s config into k8s.yaml
- add bootstrap_node, manage server and token fields in rke2 config
- manage install of helm
- manage node attributes (from puppet facts)
- manage frr exclusions for service/cluster network
This commit is contained in:
Ben Vincent
2025-09-06 23:01:57 +10:00
parent 0665873dc8
commit d95883f5ee
23 changed files with 697 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hieradata/common.yaml
+3
View File
@@ -155,6 +155,9 @@ lookup_options:
zfs::datasets:
merge:
strategy: deep
rke2::config_hash:
merge:
strategy: deep
facts_path: '/opt/puppetlabs/facter/facts.d'
hieradata/roles/infra/k8s.eyaml
+1
View File
@@ -0,0 +1 @@
rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEACQ4IIE9qT7x+KBTtvksUwvko6U/wnRn4PZEHUc91u9StIcjd0N6uH3hrJ4ddJmwikXf9Sz/5t+kSPvcVewFcjzvMHzJXqaE57KX0hqjFpqSGMlqWy4hdrZTHm/W65PkB73j21HLn9gH7m4rmqLomSbIe21P7sFM5+1pbnQejx9Us43Mu7PNy+8rQCcdEeBnRmzLaVWR8c35aXMHANqefLaLPCWjTbDY4jhcHPkNse5thpyUW89Bnq6caTJjNNQRI9U2FX2qHqwFh/rcjtGwDk9JUJvpF1qb9mdY68txMfaQ2KQh0gH67nxVf0cDppUxJaQ86Uu71BQX54l/om9AFrDCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQFfKTlWnJq2zqP6jsLAzw94BwDu32IVvfvYvsuNcOJHc4ipI0PV/6AcatIDaXjK+K8BMLdWxpOVsVf4vL1zPatB5UTwc8rm1UvxWN8SbtqF8accmAA0GbIexQezsJYaAI3NT9gItGj/aJjTitl+ed7QfNCd36HVH9FEmfDaGwN7f/yw==]
hieradata/roles/infra/k8s.yaml
+169
View File
@@ -0,0 +1,169 @@
---
hiera_include:
- profiles::selinux::setenforce
- profiles::ceph::node
- profiles::ceph::client
- exporters::frr_exporter
- frrouting
- rke2
# manage rke2
rke2::bootstrap_node: prodnxsr0001.main.unkin.net
rke2::join_url: https://join-k8s.service.consul:9345
rke2::config_hash:
bind-address: "%{hiera('networking_loopback0_ip')}"
node-ip: "%{hiera('networking_loopback0_ip')}"
node-external-ip: "%{hiera('networking_loopback0_ip')}"
write-kubeconfig-mode: 644
kubelet-arg:
- '--node-status-update-frequency=4s'
- '--max-pods=100'
node-label:
- "region=%{facts.region}"
- "country=%{facts.country}"
- "asset=%{facts.dmi.product.serial_number}"
- "zone=%{zone}"
- "environment=%{environment}"
# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
python::manage_dev_package: false
profiles::packages::include:
bridge-utils: {}
cephadm: {}
profiles::selinux::setenforce::mode: disabled
profiles::ceph::client::manage_ceph_conf: false
profiles::ceph::client::manage_ceph_package: false
profiles::ceph::client::manage_ceph_paths: false
profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
profiles::ceph::client::mons:
- 198.18.23.9
- 198.18.23.10
- 198.18.23.11
- 198.18.23.12
- 198.18.23.13
# additional repos
profiles::yum::global::repos:
ceph:
name: ceph
descr: ceph repository
target: /etc/yum.repos.d/ceph.repo
baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
gpgkey: https://download.ceph.com/keys/release.asc
mirrorlist: absent
ceph-noarch:
name: ceph-noarch
descr: ceph-noarch repository
target: /etc/yum.repos.d/ceph-noarch.repo
baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
gpgkey: https://download.ceph.com/keys/release.asc
mirrorlist: absent
frr-extras:
name: frr-extras
descr: frr-extras repository
target: /etc/yum.repos.d/frr-extras.repo
baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
frr-stable:
name: frr-stable
descr: frr-stable repository
target: /etc/yum.repos.d/frr-stable.repo
baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
rancher-rke2-common-latest:
name: rancher-rke2-common-latest
descr: rancher-rke2-common-latest
target: /etc/yum.repos.d/rke2-common.repo
baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
rancher-rke2-1-33-latest:
name: rancher-rke2-1-33-latest
descr: rancher-rke2-1-33-latest
target: /etc/yum.repos.d/rke2-1-33.repo
baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
# dns
profiles::dns::base::primary_interface: loopback0
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
"%{hiera('networking_1000_iface')}":
type: physical
ipaddress: "%{hiera('networking_1000_ip')}"
gateway: 198.18.15.254
txqueuelen: 10000
forwarding: true
"%{hiera('networking_2500_iface')}":
type: physical
ipaddress: "%{hiera('networking_2500_ip')}"
mtu: 1500
txqueuelen: 10000
forwarding: true
loopback0:
type: dummy
ipaddress: "%{hiera('networking_loopback0_ip')}"
netmask: 255.255.255.255
mtu: 1500
loopback1:
type: dummy
ipaddress: "%{hiera('networking_loopback1_ip')}"
netmask: 255.255.255.255
mtu: 1500
loopback2:
type: dummy
ipaddress: "%{hiera('networking_loopback2_ip')}"
netmask: 255.255.255.255
mtu: 1500
# configure consul service
profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
profiles::consul::client::node_rules:
- resource: service
segment: frr_exporter
disposition: write
# frrouting
exporters::frr_exporter::enable: true
frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
frrouting::ospf_preferred_source_enable: true
frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
frrouting::ospfd_redistribute:
- connected
frrouting::ospfd_interfaces:
"%{hiera('networking_1000_iface')}":
area: 0.0.0.0
"%{hiera('networking_2500_iface')}":
area: 0.0.0.0
loopback0:
area: 0.0.0.0
loopback1:
area: 0.0.0.0
loopback2:
area: 0.0.0.0
frrouting::daemons:
ospfd: true
frrouting::ospf_exclude_k8s_enable: true
frrouting::k8s_cluster_cidr: '10.42.0.0/16' # RKE2 cluster-cidr (pods)
frrouting::k8s_service_cidr: '10.43.0.0/16' # RKE2 service-cidr
# add loopback interfaces to ssh list
ssh::server::options:
ListenAddress:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
profiles::ssh::sign::principals:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
hieradata/roles/infra/k8s/compute.yaml
+2 -9
View File
@@ -1,10 +1,3 @@
---
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
eth0:
type: physical
forwarding: true
dhcp: true
mtu: 1500
# manage rke2
rke2::node_type: agent
hieradata/roles/infra/k8s/control.yaml
+64 -29
View File
@@ -1,42 +1,77 @@
---
profiles::pki::vault::alt_names:
- k8s-control.service.consul
- k8s-control.query.consul
- "k8s-control.service.%{facts.country}-%{facts.region}.consul"
profiles::ssh::sign::principals:
- k8s-control.service.consul
- k8s-control.query.consul
- "k8s-control.service.%{facts.country}-%{facts.region}.consul"
# manage rke2
rke2::node_type: server
rke2::helm_install: true
rke2::helm_repos:
metallb: https://metallb.github.io/metallb
rancher-stable: https://releases.rancher.com/server-charts/stable
rke2::extra_config_files:
- rke2-canal-config
# - rke2-cilium-config
# - rke2-metallb-pools
# - rke2-metallb-bgppeers
# - rke2-metallb-bgpadvertisements
rke2::config_hash:
advertise-address: "%{hiera('networking_loopback0_ip')}"
cluster-domain: "svc.k8s.unkin.net"
tls-san:
- "join-k8s.service.consul"
- "api-k8s.service.consul"
- "api.k8s.unkin.net"
- "join.k8s.unkin.net"
cni: canal
cluster-cidr: 10.42.0.0/16
service-cidr: 10.43.0.0/16
cluster-dns: 10.43.0.10
etcd-arg: "--quota-backend-bytes 2048000000"
etcd-snapshot-schedule-cron: "0 3 * * *"
etcd-snapshot-retention: 10
kube-apiserver-arg:
- '--default-not-ready-toleration-seconds=30'
- '--default-unreachable-toleration-seconds=30'
kube-controller-manager-arg:
- '--node-monitor-period=4s'
protect-kernel-defaults: true
# configure consul service
consul::services:
k8s-control:
service_name: 'k8s-control'
tags:
- 'k8s'
- 'container'
api-k8s:
service_name: 'api-k8s'
address: "%{facts.networking.fqdn}"
port: 6443
checks:
- id: 'k8s-control_https_check'
name: 'k8s-control HTTPS Check'
http: "https://%{facts.networking.fqdn}:6443"
method: 'GET'
tls_skip_verify: true
- id: 'api-k8s_livez_check'
name: 'api-k8s livez Check'
args:
- sudo
- /usr/local/bin/check_k8s_api.sh
interval: '10s'
timeout: '1s'
join-k8s:
service_name: 'join-k8s'
address: "%{facts.networking.fqdn}"
port: 9345
checks:
- id: 'rke2_tcp_check_9345'
name: 'rke2 TCP Check 9345'
tcp: "%{hiera('networking_loopback0_ip')}:9345"
interval: '10s'
timeout: '1s'
profiles::consul::client::node_rules:
- resource: service
segment: k8s-control
segment: api-k8s
disposition: write
- resource: service
segment: join-k8s
disposition: write
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
eth0:
type: physical
forwarding: true
dhcp: true
mtu: 1500
profiles::pki::vault::alt_names:
- api-k8s.service.consul
- api-k8s.query.consul
- "api-k8s.service.%{facts.country}-%{facts.region}.consul"
sudo::configs:
consul-checks:
priority: 20
content: |
consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh
hieradata/roles/infra/k8s/node.yaml
+68 -1
View File
@@ -5,6 +5,24 @@ hiera_include:
- profiles::ceph::node
- profiles::ceph::client
- exporters::frr_exporter
- profiles::rke2::node
# manage rke2
profiles::rke2::node::servers:
- prodnxsr0001.main.unkin.net
- prodnxsr0002.main.unkin.net
- prodnxsr0003.main.unkin.net
rke2::config_hash:
bind-address: "%{hiera('networking_loopback0_ip')}"
advertise-address: "%{hiera('networking_loopback0_ip')}"
node-ip: "%{hiera('networking_loopback0_ip')}"
node-external-ip: "%{hiera('networking_loopback0_ip')}"
cluster-domain: "svc.k8s.unkin.net"
tls-san:
- "api.k8s.unkin.net"
- "join.k8s.unkin.net"
cni: cilium
# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
python::manage_dev_package: false
@@ -25,6 +43,7 @@ profiles::ceph::client::mons:
- 198.18.23.11
- 198.18.23.12
- 198.18.23.13
# additional repos
profiles::yum::global::repos:
ceph:
@@ -55,6 +74,20 @@ profiles::yum::global::repos:
baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
rancher-rke2-common-latest:
name: rancher-rke2-common-latest
descr: rancher-rke2-common-latest
target: /etc/yum.repos.d/rke2-common.repo
baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
rancher-rke2-1-33-latest:
name: rancher-rke2-1-33-latest
descr: rancher-rke2-1-33-latest
target: /etc/yum.repos.d/rke2-1-33.repo
baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
# dns
profiles::dns::base::primary_interface: loopback0
@@ -91,9 +124,38 @@ networking::interfaces:
netmask: 255.255.255.255
mtu: 1500
# consul
# configure consul service
profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
consul::services:
api-k8s:
service_name: 'api-k8s'
address: "%{facts.networking.fqdn}"
port: 6443
checks:
- id: 'api-k8s_https_check'
name: 'api-k8s HTTPS Check'
http: "https://%{facts.networking.fqdn}:6443"
method: 'GET'
tls_skip_verify: true
interval: '10s'
timeout: '1s'
join-k8s:
service_name: 'join-k8s'
address: "%{facts.networking.fqdn}"
port: 9345
checks:
- id: 'etcd_tcp_check_9345'
name: 'ETCD TCP Check 9345'
tcp: "%{facts.networking.fqdn}:9345"
interval: '10s'
timeout: '1s'
profiles::consul::client::node_rules:
- resource: service
segment: api-k8s
disposition: write
- resource: service
segment: join-k8s
disposition: write
- resource: service
segment: frr_exporter
disposition: write
@@ -130,3 +192,8 @@ profiles::ssh::sign::principals:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
profiles::pki::vault::alt_names:
- api-k8s.service.consul
- api-k8s.query.consul
- "api-k8s.service.%{facts.country}-%{facts.region}.consul"