diff --git a/hieradata/roles/infra/k8s.eyaml b/hieradata/roles/infra/k8s.eyaml index 56f2d4b..78340d6 100644 --- a/hieradata/roles/infra/k8s.eyaml +++ b/hieradata/roles/infra/k8s.eyaml @@ -1 +1 @@ -rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAPRBcsDJv/SDR4pOi4fBWzU2ayCXW8BaATzpg3h8mHsVPcHwhfL9w9d4BIshM5yzSZwfmj8EBMk6sa4vEcBRN5r5vWDETnjwsvtLULvyiPm5WVYAYe+Z/fzoCUqswWvXrU/yzoMd/oUYFOrtCB0wT4bZX2leg64CQ7DZIf4wZod0Z9EH7YwUpL4mmbLWY29w0Z3F702O8v0hp9J7IWX5A5ob8OLaTy9SmApZEtm35UDz+OAaJ1OZ9yc2pWTkVZ0dSgMvouynJWToZ/k0ZwoW5I32WLeZ6mkYODwXF1ULqVTX3eX/c9HlVilHJar9TlcY46+8ypV3Uoju8j6c86DaRtjCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQe3h1bWn1G2HYEwupc7n3EoBwzcA4ICV4efMd1n4d1CW+wlBAoa5T8lXSluf09vajUQmaFhl3pl99v3uc5M38ePPd5QkzoZPfb4kLcmuufGNfwbIXpaQQ+nwQATOX1UG5SOEovshvWnW0iF5JrgIa+MLed9WF48NMbvZZLbwyBe7hrQ==] +rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAOD+w5nJFqEYWFj+tZQ65Oi19eDhaWtpLQ0gwEdBtMmY9sPJ63l1q2qH933NH6TOd1UlMDvGfoDLCae+yt/MAW//cJ15X3QbiVQ23DdfCOlUEZN6fjVrveEt/yIeFQmWvnkMS4pRfPbgQu2OHm37PpuPE7s6dUyGItAYjchrRhtQ7ibhXDnN7miG+oVRXP2T8b/V5WPdmA222DSV6r/AnqaWkna9W/oh/I2sNKeEm5q3f8bh8Gxt1dDy3VwaZ3lAh3uR+SUm7P/6PTYw8opxiumFBvos0mRiXIdOwUuqrAS8hafWBhxnDLlTBfz62Nc4wQmQ8gz0bHJZSipH9G6mIEDCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ2WG6ROWFlQdXx0TuO5oABoBwTDtAXIj7y6I1B3zCnFoMHETf5d7ulPGdgwZsENf0UIHpg2l0w503MUHHbu6YDFiDiTE0oDNJPVHid7TO+XwWFgh5v1MWi/XeEBgCs6nMCW8qkX0Z3UXaZdSBUll1M4sRtuqscBnoD/LLs2kKfxrqQg==] diff --git a/hieradata/roles/infra/k8s/control.yaml b/hieradata/roles/infra/k8s/control.yaml index 28a130e..bf20985 100644 --- a/hieradata/roles/infra/k8s/control.yaml +++ b/hieradata/roles/infra/k8s/control.yaml @@ -28,10 +28,9 @@ rke2::csi_ceph_templates: - ceph-csi-secret rke2::extra_config_files: - rke2-canal-config - - service-loadbalancer-nginx + - rke2-nginx-ingress-config rke2::config_hash: advertise-address: "%{hiera('networking_loopback0_ip')}" - cluster-domain: "svc.k8s.unkin.net" tls-san: - "join-k8s.service.consul" - "api-k8s.service.consul" diff --git a/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml b/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml index 8bd065f..d51189a 100644 --- a/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml +++ b/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml @@ -45,7 +45,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: csi-rbdplugin - image: quay.io/cephcsi/cephcsi:v3.15 + image: quay.io/cephcsi/cephcsi:v3.15.0 args: - "--nodeid=$(NODE_ID)" - "--type=rbd" diff --git a/modules/rke2/files/ceph-csi-rbdplugin.yaml b/modules/rke2/files/ceph-csi-rbdplugin.yaml index 3e09c87..a039e2a 100644 --- a/modules/rke2/files/ceph-csi-rbdplugin.yaml +++ b/modules/rke2/files/ceph-csi-rbdplugin.yaml @@ -25,7 +25,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: quay.io/cephcsi/cephcsi:v3.15 + image: quay.io/cephcsi/cephcsi:v3.15.0 args: - "--nodeid=$(NODE_ID)" - "--pluginpath=/var/lib/kubelet/plugins" diff --git a/modules/rke2/files/ingress-route-rancher.yaml b/modules/rke2/files/ingress-route-rancher.yaml index 1797c17..eb189f3 100644 --- a/modules/rke2/files/ingress-route-rancher.yaml +++ b/modules/rke2/files/ingress-route-rancher.yaml @@ -1,3 +1,4 @@ +--- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/modules/rke2/files/rke2-nginx-ingress-config.yaml b/modules/rke2/files/rke2-nginx-ingress-config.yaml new file mode 100644 index 0000000..c51271e --- /dev/null +++ b/modules/rke2/files/rke2-nginx-ingress-config.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-ingress-nginx + namespace: kube-system +spec: + valuesContent: |- + controller: + hostPort: + enabled: false + service: + enabled: true + type: LoadBalancer + externalTrafficPolicy: Local + loadBalancerClass: purelb.io/purelb + allocateLoadBalancerNodePorts: false + annotations: + purelb.io/service-group: common + purelb.io/addresses: "198.18.200.0" diff --git a/modules/rke2/files/service-loadbalancer-nginx.yaml b/modules/rke2/files/service-loadbalancer-nginx.yaml deleted file mode 100644 index 4615dc1..0000000 --- a/modules/rke2/files/service-loadbalancer-nginx.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rke2-ingress-nginx-controller - namespace: kube-system - annotations: - purelb.io/service-group: common -spec: - type: LoadBalancer - externalTrafficPolicy: Cluster - ports: - - name: http - port: 80 - targetPort: http - protocol: TCP - - name: https - port: 443 - targetPort: https - protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/component: controller - app.kubernetes.io/instance: rke2-ingress-nginx - loadBalancerIP: 198.18.200.0 ---- -apiVersion: helm.cattle.io/v1 -kind: HelmChartConfig -metadata: - name: rke2-ingress-nginx - namespace: kube-system -spec: - valuesContent: |- - controller: - hostPort: - enabled: false - service: - enabled: true - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - purelb.io/service-group: common diff --git a/modules/rke2/lib/facter/k8s_masters.rb b/modules/rke2/lib/facter/k8s_masters.rb new file mode 100644 index 0000000..33b9c84 --- /dev/null +++ b/modules/rke2/lib/facter/k8s_masters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'json' +require 'open3' + +Facter.add(:k8s_masters) do + confine do + File.exist?('/etc/rancher/rke2/rke2.yaml') && + File.executable?('/usr/bin/kubectl') + end + + setcode do + env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' } + cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json'] + + stdout, stderr, status = Open3.capture3(env, *cmd) + + if status.success? + json = JSON.parse(stdout) + + master_count = json['items'].count do |item| + roles = item.dig('metadata', 'labels') || {} + + # Look for well-known labels assigned to control-plane nodes + roles.any? do |key, _| + key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)} + end + end + + master_count + else + Facter.debug("kubectl error: #{stderr}") + 0 + end + rescue StandardError => e + Facter.debug("Exception in k8s_masters fact: #{e.message}") + 0 + end +end diff --git a/modules/rke2/manifests/config.pp b/modules/rke2/manifests/config.pp index e4affb7..51d499d 100644 --- a/modules/rke2/manifests/config.pp +++ b/modules/rke2/manifests/config.pp @@ -24,9 +24,7 @@ class rke2::config ( token => $node_token, } ) }else{ - $config = merge($config_hash, { - token => $node_token, - } ) + $config = merge($config_hash, {}) } } elsif $node_type == 'agent' { $config = merge($config_hash, { @@ -75,7 +73,7 @@ class rke2::config ( } # on the controller nodes only - if $node_type == 'server' { + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 { # wait for purelb helm to setup namespace if 'purelb' in $facts['k8s_namespaces'] { diff --git a/modules/rke2/manifests/helm.pp b/modules/rke2/manifests/helm.pp index e172f2e..8eb5536 100644 --- a/modules/rke2/manifests/helm.pp +++ b/modules/rke2/manifests/helm.pp @@ -20,8 +20,8 @@ class rke2::helm ( mode => '0755', } - # on the controller nodes only - if $node_type == 'server' { + # on the controller nodes only, and after 3 master nodes exist + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 { # check if the repo already exists $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {