feat: moderate the k8s install

- only install a base config
- wait for 3 masters before deploying helm charts
- remove nginx and rancher-ingres configurations

hieradata/roles/infra/k8s/control.yaml

@@ -28,7 +28,7 @@ rke2::csi_ceph_templates:
   - ceph-csi-secret
 rke2::extra_config_files:
   - rke2-canal-config
-  - service-loadbalancer-nginx
+#  - service-loadbalancer-nginx
 rke2::config_hash:
   advertise-address: "%{hiera('networking_loopback0_ip')}"
   cluster-domain: "svc.k8s.unkin.net"

modules/rke2/lib/facter/k8s_masters.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+Facter.add(:k8s_masters) do
+  confine do
+    File.exist?('/etc/rancher/rke2/rke2.yaml') &&
+      File.executable?('/usr/bin/kubectl')
+  end
+
+  setcode do
+    env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' }
+    cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json']
+
+    stdout, stderr, status = Open3.capture3(env, *cmd)
+
+    if status.success?
+      json = JSON.parse(stdout)
+
+      master_count = json['items'].count do |item|
+        roles = item.dig('metadata', 'labels') || {}
+
+        # Look for well-known labels assigned to control-plane nodes
+        roles.any? do |key, _|
+          key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)}
+        end
+      end
+
+      master_count
+    else
+      Facter.debug("kubectl error: #{stderr}")
+      0
+    end
+  rescue StandardError => e
+    Facter.debug("Exception in k8s_masters fact: #{e.message}")
+    0
+  end
+end

modules/rke2/manifests/config.pp

@@ -75,7 +75,7 @@ class rke2::config (
   }
 
   # on the controller nodes only
-  if $node_type == 'server' {
+  if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 0 {
 
     # wait for purelb helm to setup namespace
     if 'purelb' in $facts['k8s_namespaces'] {
@@ -89,17 +89,17 @@ class rke2::config (
       }
     }
 
-    # wait for rancher helm to setup namespace
-    if 'cattle-system' in $facts['k8s_namespaces'] {
-      file {'/var/lib/rancher/rke2/server/manifests/ingress-route-rancher.yaml':
-        ensure  => file,
-        owner   => 'root',
-        group   => 'root',
-        mode    => '0644',
-        source  => 'puppet:///modules/rke2/ingress-route-rancher.yaml',
-        require => Service['rke2-server'],
-      }
-    }
+    ## wait for rancher helm to setup namespace
+    #if 'cattle-system' in $facts['k8s_namespaces'] {
+    #  file {'/var/lib/rancher/rke2/server/manifests/ingress-route-rancher.yaml':
+    #    ensure  => file,
+    #    owner   => 'root',
+    #    group   => 'root',
+    #    mode    => '0644',
+    #    source  => 'puppet:///modules/rke2/ingress-route-rancher.yaml',
+    #    require => Service['rke2-server'],
+    #  }
+    #}
 
     # manage extra config config (these are not dependent on helm)
     $extra_config_files.each |$file| {

modules/rke2/manifests/helm.pp

@@ -20,8 +20,8 @@ class rke2::helm (
       mode   => '0755',
     }
 
-    # on the controller nodes only
-    if $node_type == 'server' {
+    # on the controller nodes only, and after 3 master nodes exist
+    if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 0 {
 
       # check if the repo already exists
       $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {