unkin/puppet-prod
3
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: adding rke2

Browse Source 
- manage rke2 repos
- add rke2 module (init, params, install, config, service)
- exclude setting ips for cilium interfaces
- split roles::infra::k8s::node -> control/compute roles
- moved common k8s config into k8s.yaml
- add bootstrap_node, manage server and token fields in rke2 config
- manage install of helm
This commit is contained in:
Ben Vincent
2025-09-06 23:01:57 +10:00
parent 0665873dc8
commit 34cc6e8bf9
17 changed files with 532 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hieradata/common.yaml
+3
View File
@@ -155,6 +155,9 @@ lookup_options:
zfs::datasets:
merge:
strategy: deep
rke2::config_hash:
merge:
strategy: deep
facts_path: '/opt/puppetlabs/facter/facts.d'
hieradata/roles/infra/k8s.eyaml
+1
View File
@@ -0,0 +1 @@
rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAOQ+c78bNNrJfc44CXqfmpGfyLMRmv5xsK2bKWrcwNxGTRfCEUlU7UOl7gKUI+7wHaeAhAbO0HNvxaQ+OK8lH3KRG5YumF5cLs3nrwp3PtVWKivQXns/icaYYTVpBRCaM96fMFQFqpoL2P7elMTndCtwJ/PYnN1etncRhJnV1j0Ogcnz69NZY+obuuyDozZTGojFrLg8aK46rzURi1biU5cGAI2x+CitGl8zfIcpkSwqTktlIf8kMaw8UmiR7sPk3TLh+sOqXlKVw8LzM3/fU7+x+UzE3KtdbF5mpfcnogJPwe+2wbzIkIZcXDU10MpQhaqEqzodkLmS7os5iyQPm9jCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQYn6gyEoS8w7iPQTZ/51yX4BwyKGpjdMtiBLzBBvX1wdGda8exfGTKw0TGq58R+ZZlXDfQ4sxtBszHOgmbCNKm0RS8iHZhgbtC51mG8X7zNgvD3YAXJlcVqtOb9amy9w25uFIjoSvc0G4JW1uVmbvZzgu59w6GgR0PfAw5uQrBxnkLQ==]
hieradata/roles/infra/k8s.yaml
+160
View File
@@ -0,0 +1,160 @@
---
hiera_include:
- profiles::selinux::setenforce
- profiles::ceph::node
- profiles::ceph::client
- exporters::frr_exporter
- frrouting
- rke2
# manage rke2
rke2::bootstrap_node: prodnxsr0001.main.unkin.net
rke2::join_url: https://join-k8s.service.consul:9345
rke2::config_hash:
bind-address: "%{hiera('networking_loopback0_ip')}"
node-ip: "%{hiera('networking_loopback0_ip')}"
node-external-ip: "%{hiera('networking_loopback0_ip')}"
write-kubeconfig-mode: 644
kubelet-arg:
- '--node-status-update-frequency=4s'
- '--max-pods=100'
# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
python::manage_dev_package: false
profiles::packages::include:
bridge-utils: {}
cephadm: {}
profiles::selinux::setenforce::mode: disabled
profiles::ceph::client::manage_ceph_conf: false
profiles::ceph::client::manage_ceph_package: false
profiles::ceph::client::manage_ceph_paths: false
profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
profiles::ceph::client::mons:
- 198.18.23.9
- 198.18.23.10
- 198.18.23.11
- 198.18.23.12
- 198.18.23.13
# additional repos
profiles::yum::global::repos:
ceph:
name: ceph
descr: ceph repository
target: /etc/yum.repos.d/ceph.repo
baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
gpgkey: https://download.ceph.com/keys/release.asc
mirrorlist: absent
ceph-noarch:
name: ceph-noarch
descr: ceph-noarch repository
target: /etc/yum.repos.d/ceph-noarch.repo
baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
gpgkey: https://download.ceph.com/keys/release.asc
mirrorlist: absent
frr-extras:
name: frr-extras
descr: frr-extras repository
target: /etc/yum.repos.d/frr-extras.repo
baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
frr-stable:
name: frr-stable
descr: frr-stable repository
target: /etc/yum.repos.d/frr-stable.repo
baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
rancher-rke2-common-latest:
name: rancher-rke2-common-latest
descr: rancher-rke2-common-latest
target: /etc/yum.repos.d/rke2-common.repo
baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
rancher-rke2-1-33-latest:
name: rancher-rke2-1-33-latest
descr: rancher-rke2-1-33-latest
target: /etc/yum.repos.d/rke2-1-33.repo
baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
# dns
profiles::dns::base::primary_interface: loopback0
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
"%{hiera('networking_1000_iface')}":
type: physical
ipaddress: "%{hiera('networking_1000_ip')}"
gateway: 198.18.15.254
txqueuelen: 10000
forwarding: true
"%{hiera('networking_2500_iface')}":
type: physical
ipaddress: "%{hiera('networking_2500_ip')}"
mtu: 1500
txqueuelen: 10000
forwarding: true
loopback0:
type: dummy
ipaddress: "%{hiera('networking_loopback0_ip')}"
netmask: 255.255.255.255
mtu: 1500
loopback1:
type: dummy
ipaddress: "%{hiera('networking_loopback1_ip')}"
netmask: 255.255.255.255
mtu: 1500
loopback2:
type: dummy
ipaddress: "%{hiera('networking_loopback2_ip')}"
netmask: 255.255.255.255
mtu: 1500
# configure consul service
profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
profiles::consul::client::node_rules:
- resource: service
segment: frr_exporter
disposition: write
# frrouting
exporters::frr_exporter::enable: true
frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
frrouting::ospf_preferred_source_enable: true
frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
frrouting::ospfd_redistribute:
- connected
frrouting::ospfd_interfaces:
"%{hiera('networking_1000_iface')}":
area: 0.0.0.0
"%{hiera('networking_2500_iface')}":
area: 0.0.0.0
loopback0:
area: 0.0.0.0
loopback1:
area: 0.0.0.0
loopback2:
area: 0.0.0.0
frrouting::daemons:
ospfd: true
# add loopback interfaces to ssh list
ssh::server::options:
ListenAddress:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
profiles::ssh::sign::principals:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
hieradata/roles/infra/k8s/compute.yaml
+2 -9
View File
@@ -1,10 +1,3 @@
---
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
eth0:
type: physical
forwarding: true
dhcp: true
mtu: 1500
# manage rke2
rke2::node_type: agent
hieradata/roles/infra/k8s/control.yaml
+62 -29
View File
@@ -1,42 +1,75 @@
---
profiles::pki::vault::alt_names:
- k8s-control.service.consul
- k8s-control.query.consul
- "k8s-control.service.%{facts.country}-%{facts.region}.consul"
profiles::ssh::sign::principals:
- k8s-control.service.consul
- k8s-control.query.consul
- "k8s-control.service.%{facts.country}-%{facts.region}.consul"
# manage rke2
rke2::node_type: server
rke2::helm_install: true
rke2::helm_repos:
metallb: https://metallb.github.io/metallb
rancher-stable: https://releases.rancher.com/server-charts/stable
rke2::config_hash:
advertise-address: "%{hiera('networking_loopback0_ip')}"
cluster-domain: "svc.k8s.unkin.net"
tls-san:
- "join-k8s.service.consul"
- "api-k8s.service.consul"
- "api.k8s.unkin.net"
- "join.k8s.unkin.net"
cni: cilium
cluster-cidr: 10.42.0.0/16
service-cidr: 10.43.0.0/16
cluster-dns: 10.43.0.10
etcd-arg: "--quota-backend-bytes 2048000000"
etcd-snapshot-schedule-cron: "0 3 * * *"
etcd-snapshot-retention: 10
disable:
- rke2-ingress-nginx
disable-kube-proxy: true
kube-apiserver-arg:
- '--default-not-ready-toleration-seconds=30'
- '--default-unreachable-toleration-seconds=30'
kube-controller-manager-arg:
- '--node-monitor-period=4s'
egress-selector-mode: disabled
protect-kernel-defaults: true
# configure consul service
consul::services:
k8s-control:
service_name: 'k8s-control'
tags:
- 'k8s'
- 'container'
api-k8s:
service_name: 'api-k8s'
address: "%{facts.networking.fqdn}"
port: 6443
checks:
- id: 'k8s-control_https_check'
name: 'k8s-control HTTPS Check'
http: "https://%{facts.networking.fqdn}:6443"
method: 'GET'
tls_skip_verify: true
- id: 'api-k8s_livez_check'
name: 'api-k8s livez Check'
args:
- sudo
- /usr/local/bin/check_k8s_api.sh
interval: '10s'
timeout: '1s'
join-k8s:
service_name: 'join-k8s'
address: "%{facts.networking.fqdn}"
port: 9345
checks:
- id: 'rke2_tcp_check_9345'
name: 'rke2 TCP Check 9345'
tcp: "%{hiera('networking_loopback0_ip')}:9345"
interval: '10s'
timeout: '1s'
profiles::consul::client::node_rules:
- resource: service
segment: k8s-control
segment: api-k8s
disposition: write
- resource: service
segment: join-k8s
disposition: write
# networking
systemd::manage_networkd: true
systemd::manage_all_network_files: true
networking::interfaces:
eth0:
type: physical
forwarding: true
dhcp: true
mtu: 1500
profiles::pki::vault::alt_names:
- api-k8s.service.consul
- api-k8s.query.consul
- "api-k8s.service.%{facts.country}-%{facts.region}.consul"
sudo::configs:
consul-checks:
priority: 20
content: |
consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh
hieradata/roles/infra/k8s/node.yaml
+68 -1
View File
@@ -5,6 +5,24 @@ hiera_include:
- profiles::ceph::node
- profiles::ceph::client
- exporters::frr_exporter
- profiles::rke2::node
# manage rke2
profiles::rke2::node::servers:
- prodnxsr0001.main.unkin.net
- prodnxsr0002.main.unkin.net
- prodnxsr0003.main.unkin.net
rke2::config_hash:
bind-address: "%{hiera('networking_loopback0_ip')}"
advertise-address: "%{hiera('networking_loopback0_ip')}"
node-ip: "%{hiera('networking_loopback0_ip')}"
node-external-ip: "%{hiera('networking_loopback0_ip')}"
cluster-domain: "svc.k8s.unkin.net"
tls-san:
- "api.k8s.unkin.net"
- "join.k8s.unkin.net"
cni: cilium
# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
python::manage_dev_package: false
@@ -25,6 +43,7 @@ profiles::ceph::client::mons:
- 198.18.23.11
- 198.18.23.12
- 198.18.23.13
# additional repos
profiles::yum::global::repos:
ceph:
@@ -55,6 +74,20 @@ profiles::yum::global::repos:
baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
mirrorlist: absent
rancher-rke2-common-latest:
name: rancher-rke2-common-latest
descr: rancher-rke2-common-latest
target: /etc/yum.repos.d/rke2-common.repo
baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
rancher-rke2-1-33-latest:
name: rancher-rke2-1-33-latest
descr: rancher-rke2-1-33-latest
target: /etc/yum.repos.d/rke2-1-33.repo
baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
gpgkey: https://rpm.rancher.io/public.key
mirrorlist: absent
# dns
profiles::dns::base::primary_interface: loopback0
@@ -91,9 +124,38 @@ networking::interfaces:
netmask: 255.255.255.255
mtu: 1500
# consul
# configure consul service
profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
consul::services:
api-k8s:
service_name: 'api-k8s'
address: "%{facts.networking.fqdn}"
port: 6443
checks:
- id: 'api-k8s_https_check'
name: 'api-k8s HTTPS Check'
http: "https://%{facts.networking.fqdn}:6443"
method: 'GET'
tls_skip_verify: true
interval: '10s'
timeout: '1s'
join-k8s:
service_name: 'join-k8s'
address: "%{facts.networking.fqdn}"
port: 9345
checks:
- id: 'etcd_tcp_check_9345'
name: 'ETCD TCP Check 9345'
tcp: "%{facts.networking.fqdn}:9345"
interval: '10s'
timeout: '1s'
profiles::consul::client::node_rules:
- resource: service
segment: api-k8s
disposition: write
- resource: service
segment: join-k8s
disposition: write
- resource: service
segment: frr_exporter
disposition: write
@@ -130,3 +192,8 @@ profiles::ssh::sign::principals:
- "%{hiera('networking_loopback0_ip')}"
- "%{hiera('networking_1000_ip')}"
- "%{hiera('networking_2500_ip')}"
profiles::pki::vault::alt_names:
- api-k8s.service.consul
- api-k8s.query.consul
- "api-k8s.service.%{facts.country}-%{facts.region}.consul"