diff --git a/hieradata/roles/infra/k8s/control.eyaml b/hieradata/roles/infra/k8s/control.eyaml new file mode 100644 index 0000000..c5dc244 --- /dev/null +++ b/hieradata/roles/infra/k8s/control.eyaml @@ -0,0 +1 @@ +rke::node_token: ENC[PKCS7,MIIBuQYJKoZIhvcNAQcDoIIBqjCCAaYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWU56ILL1Te14z0vG9Q9tD9DOlRfxasZHuChZlWHiw3MLOv3FwEc9+jEW20zMJCw6xoTITJ6mXN1fXe48fT3hX5jaBijoqLpfWp64zXomXCEyLh+DZn1N8ETRFS/gB6Fk1PjkCv4x4jd92tTI28gQQAJCb5oNTJzeCFJtovECSL2ElpRJKvdkf+fFtTHdpaTcX4oASz2WUyVux8oAwkGUCRlulzpAJhcwCYoKVAq1wiFRaicK/vZ9DJBlN1HrixKNCsKwRYLZYg0Qh0NjjJPI3UlrR6l06qnTvHhYAOg2BvfQIVlcUI5NjOfUD96GqTb8hFoWhsfhQ0GQT385QmsMlDB8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCj5SiNg/nNkDbK8gR+45sMgFCzjCs/vVqJ+dFQA7r7nVhejb/bKiRtWmEnnvDSN0zOVHAueCeImNBfWB3+ZDSPbXvjN6audmdmeMV+9rvq05Nxcsb5Lcawh6DUzkJWazGH3A==] diff --git a/hieradata/roles/infra/k8s/control.yaml b/hieradata/roles/infra/k8s/control.yaml index 007f267..9acfd05 100644 --- a/hieradata/roles/infra/k8s/control.yaml +++ b/hieradata/roles/infra/k8s/control.yaml @@ -1,42 +1,200 @@ --- -profiles::pki::vault::alt_names: - - k8s-control.service.consul - - k8s-control.query.consul - - "k8s-control.service.%{facts.country}-%{facts.region}.consul" +hiera_include: + - profiles::selinux::setenforce + - frrouting + - profiles::ceph::node + - profiles::ceph::client + - exporters::frr_exporter + - rke2 -profiles::ssh::sign::principals: - - k8s-control.service.consul - - k8s-control.query.consul - - "k8s-control.service.%{facts.country}-%{facts.region}.consul" +# manage rke2 +rke2::bootstrap_node: prodnxsr0001.main.unkin.net +rke2::node_type: server +rke2::config_hash: + bind-address: "%{hiera('networking_loopback0_ip')}" + advertise-address: "%{hiera('networking_loopback0_ip')}" + node-ip: "%{hiera('networking_loopback0_ip')}" + node-external-ip: "%{hiera('networking_loopback0_ip')}" + cluster-domain: "svc.k8s.unkin.net" + tls-san: + - "api.k8s.unkin.net" + - "join.k8s.unkin.net" + cni: cilium -# configure consul service -consul::services: - k8s-control: - service_name: 'k8s-control' - tags: - - 'k8s' - - 'container' - address: "%{facts.networking.fqdn}" - port: 6443 - checks: - - id: 'k8s-control_https_check' - name: 'k8s-control HTTPS Check' - http: "https://%{facts.networking.fqdn}:6443" - method: 'GET' - tls_skip_verify: true - interval: '10s' - timeout: '1s' -profiles::consul::client::node_rules: - - resource: service - segment: k8s-control - disposition: write +# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package +python::manage_dev_package: false + +profiles::packages::include: + bridge-utils: {} + cephadm: {} + +profiles::selinux::setenforce::mode: disabled + +profiles::ceph::client::manage_ceph_conf: false +profiles::ceph::client::manage_ceph_package: false +profiles::ceph::client::manage_ceph_paths: false +profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8' +profiles::ceph::client::mons: + - 198.18.23.9 + - 198.18.23.10 + - 198.18.23.11 + - 198.18.23.12 + - 198.18.23.13 + +# additional repos +profiles::yum::global::repos: + ceph: + name: ceph + descr: ceph repository + target: /etc/yum.repos.d/ceph.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture} + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent + ceph-noarch: + name: ceph-noarch + descr: ceph-noarch repository + target: /etc/yum.repos.d/ceph-noarch.repo + baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch + gpgkey: https://download.ceph.com/keys/release.asc + mirrorlist: absent + frr-extras: + name: frr-extras + descr: frr-extras repository + target: /etc/yum.repos.d/frr-extras.repo + baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os + gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR + mirrorlist: absent + frr-stable: + name: frr-stable + descr: frr-stable repository + target: /etc/yum.repos.d/frr-stable.repo + baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os + gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR + mirrorlist: absent + rancher-rke2-common-latest: + name: rancher-rke2-common-latest + descr: rancher-rke2-common-latest + target: /etc/yum.repos.d/rke2-common.repo + baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + rancher-rke2-1-33-latest: + name: rancher-rke2-1-33-latest + descr: rancher-rke2-1-33-latest + target: /etc/yum.repos.d/rke2-1-33.repo + baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64 + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + +# dns +profiles::dns::base::primary_interface: loopback0 # networking systemd::manage_networkd: true systemd::manage_all_network_files: true networking::interfaces: - eth0: + "%{hiera('networking_1000_iface')}": type: physical + ipaddress: "%{hiera('networking_1000_ip')}" + gateway: 198.18.15.254 + txqueuelen: 10000 forwarding: true - dhcp: true + "%{hiera('networking_2500_iface')}": + type: physical + ipaddress: "%{hiera('networking_2500_ip')}" mtu: 1500 + txqueuelen: 10000 + forwarding: true + loopback0: + type: dummy + ipaddress: "%{hiera('networking_loopback0_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + loopback1: + type: dummy + ipaddress: "%{hiera('networking_loopback1_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + loopback2: + type: dummy + ipaddress: "%{hiera('networking_loopback2_ip')}" + netmask: 255.255.255.255 + mtu: 1500 + +# configure consul service +profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}" +consul::services: + api-k8s: + service_name: 'api-k8s' + address: "%{facts.networking.fqdn}" + port: 6443 + checks: + - id: 'api-k8s_livez_check' + name: 'api-k8s livez Check' + args: + - sudo + - /usr/local/bin/check_k8s_api.sh + interval: '10s' + timeout: '1s' + join-k8s: + service_name: 'join-k8s' + address: "%{facts.networking.fqdn}" + port: 9345 + checks: + - id: 'rke2_tcp_check_9345' + name: 'rke2 TCP Check 9345' + tcp: "%{hiera('networking_loopback0_ip')}:9345" + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: api-k8s + disposition: write + - resource: service + segment: join-k8s + disposition: write + - resource: service + segment: frr_exporter + disposition: write + +# frrouting +exporters::frr_exporter::enable: true +frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}" +frrouting::ospfd_redistribute: + - connected +frrouting::ospfd_interfaces: + "%{hiera('networking_1000_iface')}": + area: 0.0.0.0 + "%{hiera('networking_2500_iface')}": + area: 0.0.0.0 + loopback0: + area: 0.0.0.0 + loopback1: + area: 0.0.0.0 + loopback2: + area: 0.0.0.0 +frrouting::daemons: + ospfd: true + +# add loopback interfaces to ssh list +ssh::server::options: + ListenAddress: + - "%{hiera('networking_loopback0_ip')}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" + +profiles::ssh::sign::principals: + - "%{hiera('networking_loopback0_ip')}" + - "%{hiera('networking_1000_ip')}" + - "%{hiera('networking_2500_ip')}" + +profiles::pki::vault::alt_names: + - api-k8s.service.consul + - api-k8s.query.consul + - "api-k8s.service.%{facts.country}-%{facts.region}.consul" + +sudo::configs: + consul-checks: + priority: 20 + content: | + consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh diff --git a/hieradata/roles/infra/k8s/node.yaml b/hieradata/roles/infra/k8s/node.yaml index 67e7c01..05acc36 100644 --- a/hieradata/roles/infra/k8s/node.yaml +++ b/hieradata/roles/infra/k8s/node.yaml @@ -5,6 +5,24 @@ hiera_include: - profiles::ceph::node - profiles::ceph::client - exporters::frr_exporter + - profiles::rke2::node + +# manage rke2 +profiles::rke2::node::servers: + - prodnxsr0001.main.unkin.net + - prodnxsr0002.main.unkin.net + - prodnxsr0003.main.unkin.net + +rke2::config_hash: + bind-address: "%{hiera('networking_loopback0_ip')}" + advertise-address: "%{hiera('networking_loopback0_ip')}" + node-ip: "%{hiera('networking_loopback0_ip')}" + node-external-ip: "%{hiera('networking_loopback0_ip')}" + cluster-domain: "svc.k8s.unkin.net" + tls-san: + - "api.k8s.unkin.net" + - "join.k8s.unkin.net" + cni: cilium # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package python::manage_dev_package: false @@ -25,6 +43,7 @@ profiles::ceph::client::mons: - 198.18.23.11 - 198.18.23.12 - 198.18.23.13 + # additional repos profiles::yum::global::repos: ceph: @@ -55,6 +74,20 @@ profiles::yum::global::repos: baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR mirrorlist: absent + rancher-rke2-common-latest: + name: rancher-rke2-common-latest + descr: rancher-rke2-common-latest + target: /etc/yum.repos.d/rke2-common.repo + baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent + rancher-rke2-1-33-latest: + name: rancher-rke2-1-33-latest + descr: rancher-rke2-1-33-latest + target: /etc/yum.repos.d/rke2-1-33.repo + baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64 + gpgkey: https://rpm.rancher.io/public.key + mirrorlist: absent # dns profiles::dns::base::primary_interface: loopback0 @@ -91,8 +124,37 @@ networking::interfaces: netmask: 255.255.255.255 mtu: 1500 -# consul +# configure consul service +consul::services: + api-k8s: + service_name: 'api-k8s' + address: "%{facts.networking.fqdn}" + port: 6443 + checks: + - id: 'api-k8s_https_check' + name: 'api-k8s HTTPS Check' + http: "https://%{facts.networking.fqdn}:6443" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' + join-k8s: + service_name: 'join-k8s' + address: "%{facts.networking.fqdn}" + port: 9345 + checks: + - id: 'etcd_tcp_check_9345' + name: 'ETCD TCP Check 9345' + tcp: "%{facts.networking.fqdn}:9345" + interval: '10s' + timeout: '1s' profiles::consul::client::node_rules: + - resource: service + segment: api-k8s + disposition: write + - resource: service + segment: join-k8s + disposition: write - resource: service segment: frr_exporter disposition: write @@ -127,3 +189,8 @@ profiles::ssh::sign::principals: - "%{hiera('networking_loopback0_ip')}" - "%{hiera('networking_1000_ip')}" - "%{hiera('networking_2500_ip')}" + +profiles::pki::vault::alt_names: + - api-k8s.service.consul + - api-k8s.query.consul + - "api-k8s.service.%{facts.country}-%{facts.region}.consul" diff --git a/modules/rke2/files/check_k8s_api.sh b/modules/rke2/files/check_k8s_api.sh new file mode 100644 index 0000000..2139e09 --- /dev/null +++ b/modules/rke2/files/check_k8s_api.sh @@ -0,0 +1,2 @@ +#!/usr/bin/bash +/var/lib/rancher/rke2/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml get --raw /livez diff --git a/modules/rke2/manifests/config.pp b/modules/rke2/manifests/config.pp new file mode 100644 index 0000000..09a0452 --- /dev/null +++ b/modules/rke2/manifests/config.pp @@ -0,0 +1,49 @@ +# config rke2 +class rke2::config ( + Enum['server', 'agent'] $node_type = $rke2::node_type, + Stdlib::Absolutepath $config_file = $rke2::config_file, + Hash $config_hash = $rke2::config_hash, + Stdlib::HTTPSUrl $join_url = $rke2::join_url, + Stdlib::Fqdn $bootstrap_node = $rke2::bootstrap_node, + String $node_token = $rke2::node_token, +){ + + # if its not the bootstrap node, add join path to config + if $node_type == 'server' { + if $trusted['certname'] != $bootstrap_node { + $config = merge($config_hash, { + server => $join_url, + token => $node_token, + } ) + }else{ + $config = $config_hash + } + }else{ + $config = $config_hash + } + + # create the config file + file { $config_file: + ensure => file, + content => Sensitive($config.to_yaml), + owner => 'root', + group => 'root', + mode => '0644', + } + + # create a script to verify k8s api is up (used by consul) + file {'/usr/local/bin/check_k8s_api.sh': + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/rke2/check_k8s_api.sh' + } + + # symlink kubectl to path + file {'/usr/bin/kubectl': + ensure => link, + target => '/var/lib/rancher/rke2/bin/kubectl', + } + +} diff --git a/modules/rke2/manifests/init.pp b/modules/rke2/manifests/init.pp new file mode 100644 index 0000000..e44efee --- /dev/null +++ b/modules/rke2/manifests/init.pp @@ -0,0 +1,16 @@ +# manage rke2 +class rke2 ( + Enum['server', 'agent'] $node_type = $rke2::params::node_type, + Stdlib::Absolutepath $config_file = $rke2::params::config_file, + Hash $config_hash = $rke2::params::config_hash, + Stdlib::HTTPSUrl $join_url = $rke2::params::join_url, + Stdlib::Fqdn $bootstrap_node = $rke2::params::bootstrap_node, + String $node_token = $rke2::params::node_token, +) inherits rke2::params { + + include rke2::install + include rke2::config + include rke2::service + + Class['rke2::install'] -> Class['rke2::config'] -> Class['rke2::service'] +} diff --git a/modules/rke2/manifests/install.pp b/modules/rke2/manifests/install.pp new file mode 100644 index 0000000..db4d6ce --- /dev/null +++ b/modules/rke2/manifests/install.pp @@ -0,0 +1,10 @@ +# install rke2 +class rke2::install ( + Enum['server', 'agent'] $node_type = $rke2::node_type, +){ + + package {"rke2-${node_type}": + ensure => installed, + } + +} diff --git a/modules/rke2/manifests/params.pp b/modules/rke2/manifests/params.pp new file mode 100644 index 0000000..7042736 --- /dev/null +++ b/modules/rke2/manifests/params.pp @@ -0,0 +1,9 @@ +# rke2 params +class rke2::params ( + Enum['server', 'agent'] $node_type = 'agent', + Stdlib::Absolutepath $config_file = '/etc/rancher/rke2/config.yaml', + Hash $config_hash = {}, + Stdlib::HTTPSUrl $join_url = 'https://127.0.0.1:9345', + Stdlib::Fqdn $bootstrap_node = 'localhost.localdomain', + String $node_token = '', +) {} diff --git a/modules/rke2/manifests/service.pp b/modules/rke2/manifests/service.pp new file mode 100644 index 0000000..e07c3f8 --- /dev/null +++ b/modules/rke2/manifests/service.pp @@ -0,0 +1,13 @@ +# manage rke2 service +class rke2::service ( + Enum['server', 'agent'] $node_type = $rke2::node_type, + Stdlib::Absolutepath $config_file = $rke2::config_file, +){ + + service {"rke2-${node_type}": + ensure => true, + enable => true, + subscribe => File[$config_file], + } + +} diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp index a25ba08..ee2fff0 100644 --- a/site/profiles/manifests/dns/base.pp +++ b/site/profiles/manifests/dns/base.pp @@ -47,7 +47,7 @@ class profiles::dns::base ( $facts['networking']['interfaces'].each | $interface, $data | { # exclude those without ipv4 address, lo, docker0 and anycast addresses - if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ { + if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ and $interface !~ /^cilium_/ { # use defaults for the primary_interface if $interface == $primary_interface {