feat: moderate the k8s install

- only install a base config
- wait for 3 masters before deploying helm charts
- remove cluster-domain
- manage nginx ingres via rke2 helmconfig
- fix container version for ceph
- fix oidc mount

hieradata/roles/infra/k8s.eyaml

@@ -1 +1 @@
-rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAPRBcsDJv/SDR4pOi4fBWzU2ayCXW8BaATzpg3h8mHsVPcHwhfL9w9d4BIshM5yzSZwfmj8EBMk6sa4vEcBRN5r5vWDETnjwsvtLULvyiPm5WVYAYe+Z/fzoCUqswWvXrU/yzoMd/oUYFOrtCB0wT4bZX2leg64CQ7DZIf4wZod0Z9EH7YwUpL4mmbLWY29w0Z3F702O8v0hp9J7IWX5A5ob8OLaTy9SmApZEtm35UDz+OAaJ1OZ9yc2pWTkVZ0dSgMvouynJWToZ/k0ZwoW5I32WLeZ6mkYODwXF1ULqVTX3eX/c9HlVilHJar9TlcY46+8ypV3Uoju8j6c86DaRtjCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQe3h1bWn1G2HYEwupc7n3EoBwzcA4ICV4efMd1n4d1CW+wlBAoa5T8lXSluf09vajUQmaFhl3pl99v3uc5M38ePPd5QkzoZPfb4kLcmuufGNfwbIXpaQQ+nwQATOX1UG5SOEovshvWnW0iF5JrgIa+MLed9WF48NMbvZZLbwyBe7hrQ==]
+rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAOD+w5nJFqEYWFj+tZQ65Oi19eDhaWtpLQ0gwEdBtMmY9sPJ63l1q2qH933NH6TOd1UlMDvGfoDLCae+yt/MAW//cJ15X3QbiVQ23DdfCOlUEZN6fjVrveEt/yIeFQmWvnkMS4pRfPbgQu2OHm37PpuPE7s6dUyGItAYjchrRhtQ7ibhXDnN7miG+oVRXP2T8b/V5WPdmA222DSV6r/AnqaWkna9W/oh/I2sNKeEm5q3f8bh8Gxt1dDy3VwaZ3lAh3uR+SUm7P/6PTYw8opxiumFBvos0mRiXIdOwUuqrAS8hafWBhxnDLlTBfz62Nc4wQmQ8gz0bHJZSipH9G6mIEDCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ2WG6ROWFlQdXx0TuO5oABoBwTDtAXIj7y6I1B3zCnFoMHETf5d7ulPGdgwZsENf0UIHpg2l0w503MUHHbu6YDFiDiTE0oDNJPVHid7TO+XwWFgh5v1MWi/XeEBgCs6nMCW8qkX0Z3UXaZdSBUll1M4sRtuqscBnoD/LLs2kKfxrqQg==]

hieradata/roles/infra/k8s/control.yaml

@@ -28,10 +28,9 @@ rke2::csi_ceph_templates:
   - ceph-csi-secret
 rke2::extra_config_files:
   - rke2-canal-config
-  - service-loadbalancer-nginx
+  - rke2-nginx-ingress-config
 rke2::config_hash:
   advertise-address: "%{hiera('networking_loopback0_ip')}"
-  cluster-domain: "svc.k8s.unkin.net"
   tls-san:
     - "join-k8s.service.consul"
     - "api-k8s.service.consul"

modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml

@@ -45,7 +45,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-rbdplugin
-          image: quay.io/cephcsi/cephcsi:v3.15
+          image: quay.io/cephcsi/cephcsi:v3.15.0
           args:
             - "--nodeid=$(NODE_ID)"
             - "--type=rbd"

modules/rke2/files/ceph-csi-rbdplugin.yaml

@@ -25,7 +25,7 @@ spec:
           capabilities:
             add: ["SYS_ADMIN"]
           allowPrivilegeEscalation: true
-          image: quay.io/cephcsi/cephcsi:v3.15
+          image: quay.io/cephcsi/cephcsi:v3.15.0
           args:
             - "--nodeid=$(NODE_ID)"
             - "--pluginpath=/var/lib/kubelet/plugins"
@@ -136,7 +136,13 @@ spec:
         - name: keys-tmp-dir
           emptyDir:
             medium: Memory
-
+        - name: oidc-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                path: oidc-token
+                expirationSeconds: 3607
+                audience: ceph-csi-kms
 ---
 apiVersion: v1
 kind: Service

modules/rke2/files/ingress-route-rancher.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

modules/rke2/files/rke2-nginx-ingress-config.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      hostPort:
+        enabled: false
+      service:
+        enabled: true
+        type: LoadBalancer
+        externalTrafficPolicy: Local
+        loadBalancerClass: purelb.io/purelb
+        allocateLoadBalancerNodePorts: false
+        annotations:
+          purelb.io/service-group: common
+          purelb.io/addresses: "198.18.200.0"

modules/rke2/files/service-loadbalancer-nginx.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: rke2-ingress-nginx-controller
-  namespace: kube-system
-  annotations:
-    purelb.io/service-group: common
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Cluster
-  ports:
-    - name: http
-      port: 80
-      targetPort: http
-      protocol: TCP
-    - name: https
-      port: 443
-      targetPort: https
-      protocol: TCP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/instance: rke2-ingress-nginx
-    loadBalancerIP: 198.18.200.0
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChartConfig
-metadata:
-  name: rke2-ingress-nginx
-  namespace: kube-system
-spec:
-  valuesContent: |-
-    controller:
-      hostPort:
-        enabled: false
-      service:
-        enabled: true
-        type: LoadBalancer
-        externalTrafficPolicy: Local
-        annotations:
-          purelb.io/service-group: common

modules/rke2/lib/facter/k8s_masters.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+Facter.add(:k8s_masters) do
+  confine do
+    File.exist?('/etc/rancher/rke2/rke2.yaml') &&
+      File.executable?('/usr/bin/kubectl')
+  end
+
+  setcode do
+    env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' }
+    cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json']
+
+    stdout, stderr, status = Open3.capture3(env, *cmd)
+
+    if status.success?
+      json = JSON.parse(stdout)
+
+      master_count = json['items'].count do |item|
+        roles = item.dig('metadata', 'labels') || {}
+
+        # Look for well-known labels assigned to control-plane nodes
+        roles.any? do |key, _|
+          key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)}
+        end
+      end
+
+      master_count
+    else
+      Facter.debug("kubectl error: #{stderr}")
+      0
+    end
+  rescue StandardError => e
+    Facter.debug("Exception in k8s_masters fact: #{e.message}")
+    0
+  end
+end

modules/rke2/manifests/config.pp

@@ -24,9 +24,7 @@ class rke2::config (
         token  => $node_token,
       } )
     }else{
-      $config = merge($config_hash, {
-        token  => $node_token,
-      } )
+      $config = merge($config_hash, {})
     }
   } elsif $node_type == 'agent' {
       $config = merge($config_hash, {
@@ -75,7 +73,7 @@ class rke2::config (
   }
 
   # on the controller nodes only
-  if $node_type == 'server' {
+  if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 {
 
     # wait for purelb helm to setup namespace
     if 'purelb' in $facts['k8s_namespaces'] {

modules/rke2/manifests/helm.pp

@@ -20,8 +20,8 @@ class rke2::helm (
       mode   => '0755',
     }
 
-    # on the controller nodes only
-    if $node_type == 'server' {
+    # on the controller nodes only, and after 3 master nodes exist
+    if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 {
 
       # check if the repo already exists
       $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {