feat: moderate the k8s install

- only install a base config
- wait for 3 masters before deploying helm charts
- remove cluster-domain
- manage nginx ingres via rke2 helmconfig
- fix container version for ceph
- fix oidc mount

modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml

@@ -45,7 +45,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-rbdplugin
-          image: quay.io/cephcsi/cephcsi:v3.15
+          image: quay.io/cephcsi/cephcsi:v3.15.0
           args:
             - "--nodeid=$(NODE_ID)"
             - "--type=rbd"

modules/rke2/files/ceph-csi-rbdplugin.yaml

@@ -25,7 +25,7 @@ spec:
           capabilities:
             add: ["SYS_ADMIN"]
           allowPrivilegeEscalation: true
-          image: quay.io/cephcsi/cephcsi:v3.15
+          image: quay.io/cephcsi/cephcsi:v3.15.0
           args:
             - "--nodeid=$(NODE_ID)"
             - "--pluginpath=/var/lib/kubelet/plugins"
@@ -136,7 +136,13 @@ spec:
         - name: keys-tmp-dir
           emptyDir:
             medium: Memory
-
+        - name: oidc-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                path: oidc-token
+                expirationSeconds: 3607
+                audience: ceph-csi-kms
 ---
 apiVersion: v1
 kind: Service

modules/rke2/files/ingress-route-rancher.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

modules/rke2/files/rke2-nginx-ingress-config.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      hostPort:
+        enabled: false
+      service:
+        enabled: true
+        type: LoadBalancer
+        externalTrafficPolicy: Local
+        loadBalancerClass: purelb.io/purelb
+        allocateLoadBalancerNodePorts: false
+        annotations:
+          purelb.io/service-group: common
+          purelb.io/addresses: "198.18.200.0"

modules/rke2/files/service-loadbalancer-nginx.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: rke2-ingress-nginx-controller
-  namespace: kube-system
-  annotations:
-    purelb.io/service-group: common
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Cluster
-  ports:
-    - name: http
-      port: 80
-      targetPort: http
-      protocol: TCP
-    - name: https
-      port: 443
-      targetPort: https
-      protocol: TCP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/instance: rke2-ingress-nginx
-    loadBalancerIP: 198.18.200.0
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChartConfig
-metadata:
-  name: rke2-ingress-nginx
-  namespace: kube-system
-spec:
-  valuesContent: |-
-    controller:
-      hostPort:
-        enabled: false
-      service:
-        enabled: true
-        type: LoadBalancer
-        externalTrafficPolicy: Local
-        annotations:
-          purelb.io/service-group: common