diff --git a/hieradata/roles/infra/k8s/control.yaml b/hieradata/roles/infra/k8s/control.yaml index 007f267..20c1329 100644 --- a/hieradata/roles/infra/k8s/control.yaml +++ b/hieradata/roles/infra/k8s/control.yaml @@ -1,42 +1,192 @@ --- -profiles::pki::vault::alt_names: - - k8s-control.service.consul - - k8s-control.query.consul - - "k8s-control.service.%{facts.country}-%{facts.region}.consul" +hiera_include: + - profiles::selinux::setenforce + - frrouting + - profiles::ceph::node + - profiles::ceph::client + - exporters::frr_exporter + - rke2 -profiles::ssh::sign::principals: - - k8s-control.service.consul - - k8s-control.query.consul - - "k8s-control.service.%{facts.country}-%{facts.region}.consul" +# manage rke2 +rke2::node_type: server +rke2::config_hash: + bind-address: "%{hiera('networking_loopback0_ip')}" + advertise-address: "%{hiera('networking_loopback0_ip')}" + node-ip: "%{hiera('networking_loopback0_ip')}" + node-external-ip: "%{hiera('networking_loopback0_ip')}" + cluster-domain: "svc.k8s.unkin.net" + tls-san: + - "api.k8s.unkin.net" + - "join.k8s.unkin.net" + cni: cilium -# configure consul service -consul::services: - k8s-control: - service_name: 'k8s-control' - tags: - - 'k8s' - - 'container' - address: "%{facts.networking.fqdn}" - port: 6443 - checks: - - id: 'k8s-control_https_check' - name: 'k8s-control HTTPS Check' - http: "https://%{facts.networking.fqdn}:6443" - method: 'GET' - tls_skip_verify: true - interval: '10s' - timeout: '1s' -profiles::consul::client::node_rules: - - resource: service - segment: k8s-control - disposition: write +# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package +python::manage_dev_package: false + +profiles::packages::include: + bridge-utils: {} + cephadm: {} + +profiles::selinux::setenforce::mode: disabled + +profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_package: false +profiles::ceph::client::manage_ceph_paths: false +profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' +profiles::ceph::client::mons: + - 198.18.23.9 + - 198.18.23.10 + - 198.18.23.11 + - 198.18.23.12 + - 198.18.23.13 + +# additional repos +profiles::yum::global::repos: + ceph: + name: ceph + descr: ceph repository + target: /etc/yum.repos.d/ceph.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture} + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent + ceph-noarch: + name: ceph-noarch + descr: ceph-noarch repository + target: /etc/yum.repos.d/ceph-noarch.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent + frr-extras: + name: frr-extras + descr: frr-extras repository + target: /etc/yum.repos.d/frr-extras.repo + baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os + gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR + mirrorlist: absent + frr-stable: + name: frr-stable + descr: frr-stable repository + target: /etc/yum.repos.d/frr-stable.repo + baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os + gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR + mirrorlist: absent + rancher-rke2-common-latest: + name: rancher-rke2-common-latest + descr: rancher-rke2-common-latest + target: /etc/yum.repos.d/rke2-common.repo + baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + rancher-rke2-1-33-latest: + name: rancher-rke2-1-33-latest + descr: rancher-rke2-1-33-latest + target: /etc/yum.repos.d/rke2-1-33.repo + baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64 + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + +# dns +profiles::dns::base::primary_interface: loopback0 # networking systemd::manage_networkd: true systemd::manage_all_network_files: true networking::interfaces: - eth0: + "%{hiera('networking_1000_iface')}": type: physical + ipaddress: "%{hiera('networking_1000_ip')}" + gateway: 198.18.15.254 + txqueuelen: 10000 forwarding: true - dhcp: true + "%{hiera('networking_2500_iface')}": + type: physical + ipaddress: "%{hiera('networking_2500_ip')}" mtu: 1500 + txqueuelen: 10000 + forwarding: true + loopback0: + type: dummy + ipaddress: "%{hiera('networking_loopback0_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + loopback1: + type: dummy + ipaddress: "%{hiera('networking_loopback1_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + loopback2: + type: dummy + ipaddress: "%{hiera('networking_loopback2_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + +# configure consul service +consul::services: + api-k8s: + service_name: 'api-k8s' + address: "%{facts.networking.fqdn}" + port: 6443 + checks: + - id: 'api-k8s_https_check' + name: 'api-k8s HTTPS Check' + http: "https://%{hiera('networking_loopback0_ip')}:6443" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' + join-k8s: + service_name: 'join-k8s' + address: "%{facts.networking.fqdn}" + port: 9345 + checks: + - id: 'rke2_tcp_check_9345' + name: 'rke2 TCP Check 9345' + tcp: "%{hiera('networking_loopback0_ip')}:9345" + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: api-k8s + disposition: write + - resource: service + segment: join-k8s + disposition: write + - resource: service + segment: frr_exporter + disposition: write + +# frrouting +exporters::frr_exporter::enable: true +frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}" +frrouting::ospfd_redistribute: + - connected +frrouting::ospfd_interfaces: + "%{hiera('networking_1000_iface')}": + area: 0.0.0.0 + "%{hiera('networking_2500_iface')}": + area: 0.0.0.0 + loopback0: + area: 0.0.0.0 + loopback1: + area: 0.0.0.0 + loopback2: + area: 0.0.0.0 +frrouting::daemons: + ospfd: true + +# add loopback interfaces to ssh list +ssh::server::options: + ListenAddress: + - "%{hiera('networking_loopback0_ip')}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" + +profiles::ssh::sign::principals: + - "%{hiera('networking_loopback0_ip')}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" + +profiles::pki::vault::alt_names: + - api-k8s.service.consul + - api-k8s.query.consul + - "api-k8s.service.%{facts.country}-%{facts.region}.consul" diff --git a/hieradata/roles/infra/k8s/node.yaml b/hieradata/roles/infra/k8s/node.yaml index 67e7c01..05acc36 100644 --- a/hieradata/roles/infra/k8s/node.yaml +++ b/hieradata/roles/infra/k8s/node.yaml @@ -5,6 +5,24 @@ hiera_include: - profiles::ceph::node - profiles::ceph::client - exporters::frr_exporter + - profiles::rke2::node + +# manage rke2 +profiles::rke2::node::servers: + - prodnxsr0001.main.unkin.net + - prodnxsr0002.main.unkin.net + - prodnxsr0003.main.unkin.net + +rke2::config_hash: + bind-address: "%{hiera('networking_loopback0_ip')}" + advertise-address: "%{hiera('networking_loopback0_ip')}" + node-ip: "%{hiera('networking_loopback0_ip')}" + node-external-ip: "%{hiera('networking_loopback0_ip')}" + cluster-domain: "svc.k8s.unkin.net" + tls-san: + - "api.k8s.unkin.net" + - "join.k8s.unkin.net" + cni: cilium # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package python::manage_dev_package: false @@ -25,6 +43,7 @@ profiles::ceph::client::mons: - 198.18.23.11 - 198.18.23.12 - 198.18.23.13 + # additional repos profiles::yum::global::repos: ceph: @@ -55,6 +74,20 @@ profiles::yum::global::repos: baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR mirrorlist: absent + rancher-rke2-common-latest: + name: rancher-rke2-common-latest + descr: rancher-rke2-common-latest + target: /etc/yum.repos.d/rke2-common.repo + baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + rancher-rke2-1-33-latest: + name: rancher-rke2-1-33-latest + descr: rancher-rke2-1-33-latest + target: /etc/yum.repos.d/rke2-1-33.repo + baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64 + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent # dns profiles::dns::base::primary_interface: loopback0 @@ -91,8 +124,37 @@ networking::interfaces: netmask: 255.255.255.255 mtu: 1500 -# consul +# configure consul service +consul::services: + api-k8s: + service_name: 'api-k8s' + address: "%{facts.networking.fqdn}" + port: 6443 + checks: + - id: 'api-k8s_https_check' + name: 'api-k8s HTTPS Check' + http: "https://%{facts.networking.fqdn}:6443" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' + join-k8s: + service_name: 'join-k8s' + address: "%{facts.networking.fqdn}" + port: 9345 + checks: + - id: 'etcd_tcp_check_9345' + name: 'ETCD TCP Check 9345' + tcp: "%{facts.networking.fqdn}:9345" + interval: '10s' + timeout: '1s' profiles::consul::client::node_rules: + - resource: service + segment: api-k8s + disposition: write + - resource: service + segment: join-k8s + disposition: write - resource: service segment: frr_exporter disposition: write @@ -127,3 +189,8 @@ profiles::ssh::sign::principals: - "%{hiera('networking_loopback0_ip')}" - "%{hiera('networking_1000_ip')}" - "%{hiera('networking_2500_ip')}" + +profiles::pki::vault::alt_names: + - api-k8s.service.consul + - api-k8s.query.consul + - "api-k8s.service.%{facts.country}-%{facts.region}.consul" diff --git a/modules/rke2/manifests/config.pp b/modules/rke2/manifests/config.pp new file mode 100644 index 0000000..0b32699 --- /dev/null +++ b/modules/rke2/manifests/config.pp @@ -0,0 +1,15 @@ +# config rke2 +class rke2::config ( + Enum['server', 'agent'] $node_type = $rke2::node_type, + Stdlib::Absolutepath $config_file = $rke2::config_file, + Hash $config_hash = $rke2::config_hash, +){ + + file { $config_file: + ensure => file, + content => $config_hash.to_yaml, + owner => 'root', + group => 'root', + mode => '0644', + } +} diff --git a/modules/rke2/manifests/init.pp b/modules/rke2/manifests/init.pp new file mode 100644 index 0000000..7d99d5d --- /dev/null +++ b/modules/rke2/manifests/init.pp @@ -0,0 +1,13 @@ +# manage rke2 +class rke2 ( + Enum['server', 'agent'] $node_type = $rke2::params::node_type, + Stdlib::Absolutepath $config_file = $rke2::params::config_file, + Hash $config_hash = $rke2::params::config_hash, +) inherits rke2::params { + + include rke2::install + include rke2::config + include rke2::service + + Class['rke2::install'] -> Class['rke2::config'] -> Class['rke2::service'] +} diff --git a/modules/rke2/manifests/install.pp b/modules/rke2/manifests/install.pp new file mode 100644 index 0000000..db4d6ce --- /dev/null +++ b/modules/rke2/manifests/install.pp @@ -0,0 +1,10 @@ +# install rke2 +class rke2::install ( + Enum['server', 'agent'] $node_type = $rke2::node_type, +){ + + package {"rke2-${node_type}": + ensure => installed, + } + +} diff --git a/modules/rke2/manifests/params.pp b/modules/rke2/manifests/params.pp new file mode 100644 index 0000000..280f4e4 --- /dev/null +++ b/modules/rke2/manifests/params.pp @@ -0,0 +1,6 @@ +# rke2 params +class rke2::params ( + Enum['server', 'agent'] $node_type = 'agent', + Stdlib::Absolutepath $config_file = '/etc/rancher/rke2/config.yaml', + Hash $config_hash = {}, +) {} diff --git a/modules/rke2/manifests/service.pp b/modules/rke2/manifests/service.pp new file mode 100644 index 0000000..e07c3f8 --- /dev/null +++ b/modules/rke2/manifests/service.pp @@ -0,0 +1,13 @@ +# manage rke2 service +class rke2::service ( + Enum['server', 'agent'] $node_type = $rke2::node_type, + Stdlib::Absolutepath $config_file = $rke2::config_file, +){ + + service {"rke2-${node_type}": + ensure => true, + enable => true, + subscribe => File[$config_file], + } + +} diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp index a25ba08..ee2fff0 100644 --- a/site/profiles/manifests/dns/base.pp +++ b/site/profiles/manifests/dns/base.pp @@ -47,7 +47,7 @@ class profiles::dns::base ( $facts['networking']['interfaces'].each | $interface, $data | { # exclude those without ipv4 address, lo, docker0 and anycast addresses - if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ { + if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ and $interface !~ /^cilium_/ { # use defaults for the primary_interface if $interface == $primary_interface {