diff --git a/hieradata/roles/infra/k8s/control.yaml b/hieradata/roles/infra/k8s/control.yaml index 28a130e..5d611cc 100644 --- a/hieradata/roles/infra/k8s/control.yaml +++ b/hieradata/roles/infra/k8s/control.yaml @@ -28,7 +28,7 @@ rke2::csi_ceph_templates: - ceph-csi-secret rke2::extra_config_files: - rke2-canal-config - - service-loadbalancer-nginx +# - service-loadbalancer-nginx rke2::config_hash: advertise-address: "%{hiera('networking_loopback0_ip')}" cluster-domain: "svc.k8s.unkin.net" diff --git a/modules/rke2/lib/facter/k8s_masters.rb b/modules/rke2/lib/facter/k8s_masters.rb new file mode 100644 index 0000000..33b9c84 --- /dev/null +++ b/modules/rke2/lib/facter/k8s_masters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'json' +require 'open3' + +Facter.add(:k8s_masters) do + confine do + File.exist?('/etc/rancher/rke2/rke2.yaml') && + File.executable?('/usr/bin/kubectl') + end + + setcode do + env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' } + cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json'] + + stdout, stderr, status = Open3.capture3(env, *cmd) + + if status.success? + json = JSON.parse(stdout) + + master_count = json['items'].count do |item| + roles = item.dig('metadata', 'labels') || {} + + # Look for well-known labels assigned to control-plane nodes + roles.any? do |key, _| + key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)} + end + end + + master_count + else + Facter.debug("kubectl error: #{stderr}") + 0 + end + rescue StandardError => e + Facter.debug("Exception in k8s_masters fact: #{e.message}") + 0 + end +end diff --git a/modules/rke2/manifests/config.pp b/modules/rke2/manifests/config.pp index e4affb7..40c0b87 100644 --- a/modules/rke2/manifests/config.pp +++ b/modules/rke2/manifests/config.pp @@ -75,7 +75,7 @@ class rke2::config ( } # on the controller nodes only - if $node_type == 'server' { + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 0 { # wait for purelb helm to setup namespace if 'purelb' in $facts['k8s_namespaces'] { @@ -89,17 +89,17 @@ class rke2::config ( } } - # wait for rancher helm to setup namespace - if 'cattle-system' in $facts['k8s_namespaces'] { - file {'/var/lib/rancher/rke2/server/manifests/ingress-route-rancher.yaml': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/rke2/ingress-route-rancher.yaml', - require => Service['rke2-server'], - } - } + ## wait for rancher helm to setup namespace + #if 'cattle-system' in $facts['k8s_namespaces'] { + # file {'/var/lib/rancher/rke2/server/manifests/ingress-route-rancher.yaml': + # ensure => file, + # owner => 'root', + # group => 'root', + # mode => '0644', + # source => 'puppet:///modules/rke2/ingress-route-rancher.yaml', + # require => Service['rke2-server'], + # } + #} # manage extra config config (these are not dependent on helm) $extra_config_files.each |$file| { diff --git a/modules/rke2/manifests/helm.pp b/modules/rke2/manifests/helm.pp index e172f2e..3b83cfe 100644 --- a/modules/rke2/manifests/helm.pp +++ b/modules/rke2/manifests/helm.pp @@ -20,8 +20,8 @@ class rke2::helm ( mode => '0755', } - # on the controller nodes only - if $node_type == 'server' { + # on the controller nodes only, and after 3 master nodes exist + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 0 { # check if the repo already exists $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {