feat: moderate the k8s install (#403)

- only install a base config - wait for 3 masters before deploying helm charts - remove cluster-domain - manage nginx ingres via rke2 helmconfig Reviewed-on: #403
2025-10-12 17:50:24 +11:00 · 2025-10-12 17:50:24 +11:00 · efbbb6bcb1
commit efbbb6bcb1
parent 16e654fdd7
17 changed files with 66 additions and 643 deletions
--- a/hieradata/roles/infra/k8s.eyaml
+++ b/hieradata/roles/infra/k8s.eyaml
@ -1 +1 @@
-rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAPRBcsDJv/SDR4pOi4fBWzU2ayCXW8BaATzpg3h8mHsVPcHwhfL9w9d4BIshM5yzSZwfmj8EBMk6sa4vEcBRN5r5vWDETnjwsvtLULvyiPm5WVYAYe+Z/fzoCUqswWvXrU/yzoMd/oUYFOrtCB0wT4bZX2leg64CQ7DZIf4wZod0Z9EH7YwUpL4mmbLWY29w0Z3F702O8v0hp9J7IWX5A5ob8OLaTy9SmApZEtm35UDz+OAaJ1OZ9yc2pWTkVZ0dSgMvouynJWToZ/k0ZwoW5I32WLeZ6mkYODwXF1ULqVTX3eX/c9HlVilHJar9TlcY46+8ypV3Uoju8j6c86DaRtjCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQe3h1bWn1G2HYEwupc7n3EoBwzcA4ICV4efMd1n4d1CW+wlBAoa5T8lXSluf09vajUQmaFhl3pl99v3uc5M38ePPd5QkzoZPfb4kLcmuufGNfwbIXpaQQ+nwQATOX1UG5SOEovshvWnW0iF5JrgIa+MLed9WF48NMbvZZLbwyBe7hrQ==]
+rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAOD+w5nJFqEYWFj+tZQ65Oi19eDhaWtpLQ0gwEdBtMmY9sPJ63l1q2qH933NH6TOd1UlMDvGfoDLCae+yt/MAW//cJ15X3QbiVQ23DdfCOlUEZN6fjVrveEt/yIeFQmWvnkMS4pRfPbgQu2OHm37PpuPE7s6dUyGItAYjchrRhtQ7ibhXDnN7miG+oVRXP2T8b/V5WPdmA222DSV6r/AnqaWkna9W/oh/I2sNKeEm5q3f8bh8Gxt1dDy3VwaZ3lAh3uR+SUm7P/6PTYw8opxiumFBvos0mRiXIdOwUuqrAS8hafWBhxnDLlTBfz62Nc4wQmQ8gz0bHJZSipH9G6mIEDCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ2WG6ROWFlQdXx0TuO5oABoBwTDtAXIj7y6I1B3zCnFoMHETf5d7ulPGdgwZsENf0UIHpg2l0w503MUHHbu6YDFiDiTE0oDNJPVHid7TO+XwWFgh5v1MWi/XeEBgCs6nMCW8qkX0Z3UXaZdSBUll1M4sRtuqscBnoD/LLs2kKfxrqQg==]
--- a/hieradata/roles/infra/k8s/control.eyaml
+++ b/hieradata/roles/infra/k8s/control.eyaml
@ -1,2 +1 @@
 ---
-rke2::csi_ceph_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEApQ371O4nGSrFB5tOZFTSJP+kJj3wJyEcWiNfonYA5LmbaMnQ6pUortec1519WHMICpSWdpq3O8frivm2CK3taYoKczeTzbsFTxvVp7s6gIZJUsCeqGHuq81YyjPtJE+Yy5IOBJjhe/8ECkEFNr0JlhwKBPWfTx5hHOzRdkGlN464weGFQtCI8UgdGe7AWEePG+u3e4RL+xCriw5tfuqMeeo+isDwVf30nK9NxsnmliOd/+jNW+GrtzycHAeokQOKnxfgrKll5Y5+npy5WueuSCEw1E+Io0NI/4Jthi7zu24UQu0KT8iRsqhuD5mr1ymvCNREnvCcVWt8VVRTGXQV+TBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDS8VXZM3wEAFRALB/Fa19dgDCTRYhU8YY4g9zREP7epY9x2MRWaTT84Jx9w5Dc/XWaRrmL4yL2sK+QHSy2057jHzo=]
--- a/hieradata/roles/infra/k8s/control.yaml
+++ b/hieradata/roles/infra/k8s/control.yaml
@ -9,29 +9,11 @@ rke2::helm_repos:
  harbor: https://helm.goharbor.io
  traefik: https://traefik.github.io/charts
  hashicorp: https://helm.releases.hashicorp.com
-rke2::csi_ceph_enable: true
-rke2::csi_ceph_clusterid: de96a98f-3d23-465a-a899-86d3d67edab8
-rke2::csi_ceph_poolname: kubernetes
-rke2::csi_ceph_monitors:
-  - 198.18.23.9:6789
-  - 198.18.23.10:6789
-  - 198.18.23.11:6789
-  - 198.18.23.12:6789
-  - 198.18.23.13:6789
-rke2::csi_ceph_files:
-  - ceph-csi-nodeplugin-rbac
-  - ceph-csi-provisioner-rbac
-  - ceph-csi-rbdplugin-provisioner
-  - ceph-csi-rbdplugin
-rke2::csi_ceph_templates:
-  - ceph-csi-config
-  - ceph-csi-secret
 rke2::extra_config_files:
  - rke2-canal-config
-  - service-loadbalancer-nginx
+  - rke2-nginx-ingress-config
 rke2::config_hash:
  advertise-address: "%{hiera('networking_loopback0_ip')}"
-  cluster-domain: "svc.k8s.unkin.net"
  tls-san:
    - "join-k8s.service.consul"
    - "api-k8s.service.consul"
--- a/modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml
+++ b/modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml
@ -1,48 +0,0 @@
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-nodeplugin
-  namespace: ceph-csi
-
---
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["list", "get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-
---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-nodeplugin
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-csi-nodeplugin
-  apiGroup: rbac.authorization.k8s.io
--- a/modules/rke2/files/ceph-csi-provisioner-rbac.yaml
+++ b/modules/rke2/files/ceph-csi-provisioner-rbac.yaml
@ -1,125 +0,0 @@
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-provisioner
-  namespace: ceph-csi
-
---
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-runner
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update", "patch", "create"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["get", "list", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments/status"]
-    verbs: ["patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationcontents"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationclasses"]
-    verbs: ["get", "list", "watch"]
-
---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-external-provisioner-runner
-  apiGroup: rbac.authorization.k8s.io
-
---
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-cfg
-  namespace: ceph-csi
-rules:
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "list", "watch", "create", "update", "delete"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
-
---
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role-cfg
-  namespace: ceph-csi
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: Role
-  name: rbd-external-provisioner-cfg
-  apiGroup: rbac.authorization.k8s.io
--- a/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml
+++ b/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml
@ -1,124 +0,0 @@
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  selector:
-    app: csi-rbdplugin-provisioner
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: csi-rbdplugin-provisioner
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin-provisioner
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - csi-rbdplugin-provisioner
-              topologyKey: "kubernetes.io/hostname"
-      serviceAccountName: rbd-csi-provisioner
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: csi-rbdplugin
-          image: quay.io/cephcsi/cephcsi:v3.15
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--type=rbd"
-            - "--controllerserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--pidlimit=-1"
-            - "--rbdhardmaxclonedepth=8"
-            - "--rbdsoftmaxclonedepth=4"
-            - "--enableprofiling=false"
-            - "--setmetadata=true"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi-provisioner.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-        # snapshotter & other sidecars omitted in this snippet for brevity
-      volumes:
-        - name: socket-dir
-          emptyDir:
-            medium: Memory
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-        # and other volumes as in the original
--- a/modules/rke2/files/ceph-csi-rbdplugin.yaml
+++ b/modules/rke2/files/ceph-csi-rbdplugin.yaml
@ -1,155 +0,0 @@
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: csi-rbdplugin
-  namespace: ceph-csi
-spec:
-  selector:
-    matchLabels:
-      app: csi-rbdplugin
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin
-    spec:
-      serviceAccountName: rbd-csi-nodeplugin
-      hostNetwork: true
-      hostPID: true
-      priorityClassName: system-node-critical
-      dnsPolicy: ClusterFirstWithHostNet
-      containers:
-        - name: csi-rbdplugin
-          securityContext:
-            privileged: true
-          capabilities:
-            add: ["SYS_ADMIN"]
-          allowPrivilegeEscalation: true
-          image: quay.io/cephcsi/cephcsi:v3.15
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--pluginpath=/var/lib/kubelet/plugins"
-            - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/"
-            - "--type=rbd"
-            - "--nodeserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--enableprofiling=false"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: host-mount
-              mountPath: /run/mount
-            - name: etc-selinux
-              mountPath: /etc/selinux
-              readOnly: true
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: plugin-dir
-              mountPath: /var/lib/kubelet/plugins
-              mountPropagation: "Bidirectional"
-            - name: mountpoint-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-            - name: ceph-logdir
-              mountPath: /var/log/ceph
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: oidc-token
-              mountPath: /run/secrets/tokens
-              readOnly: true
-        # possibly sidecars like driver-registrar, liveness, etc.
-      volumes:
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
-            type: DirectoryOrCreate
-        - name: plugin-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins
-            type: Directory
-        - name: mountpoint-dir
-          hostPath:
-            path: /var/lib/kubelet/pods
-            type: DirectoryOrCreate
-        - name: ceph-logdir
-          hostPath:
-            path: /var/log/ceph
-            type: DirectoryOrCreate
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: etc-selinux
-          hostPath:
-            path: /etc/selinux
-            type: DirectoryOrCreate
-        - name: host-mount
-          hostPath:
-            path: /run/mount
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-            type: DirectoryOrCreate
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-metrics-rbdplugin
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-  selector:
-    app: csi-rbdplugin
--- a/modules/rke2/files/ingress-route-rancher.yaml
+++ b/modules/rke2/files/ingress-route-rancher.yaml
@ -1,3 +1,4 @@
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
--- a/modules/rke2/files/rke2-nginx-ingress-config.yaml
+++ b/modules/rke2/files/rke2-nginx-ingress-config.yaml
@ -0,0 +1,20 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      hostPort:
+        enabled: false
+      service:
+        enabled: true
+        type: LoadBalancer
+        externalTrafficPolicy: Local
+        loadBalancerClass: purelb.io/purelb
+        allocateLoadBalancerNodePorts: false
+        annotations:
+          purelb.io/service-group: common
+          purelb.io/addresses: "198.18.200.0"
--- a/modules/rke2/files/service-loadbalancer-nginx.yaml
+++ b/modules/rke2/files/service-loadbalancer-nginx.yaml
@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: rke2-ingress-nginx-controller
-  namespace: kube-system
-  annotations:
-    purelb.io/service-group: common
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Cluster
-  ports:
-    - name: http
-      port: 80
-      targetPort: http
-      protocol: TCP
-    - name: https
-      port: 443
-      targetPort: https
-      protocol: TCP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/instance: rke2-ingress-nginx
-    loadBalancerIP: 198.18.200.0
---
-apiVersion: helm.cattle.io/v1
-kind: HelmChartConfig
-metadata:
-  name: rke2-ingress-nginx
-  namespace: kube-system
-spec:
-  valuesContent: |-
-    controller:
-      hostPort:
-        enabled: false
-      service:
-        enabled: true
-        type: LoadBalancer
-        externalTrafficPolicy: Local
-        annotations:
-          purelb.io/service-group: common
--- a/modules/rke2/lib/facter/k8s_masters.rb
+++ b/modules/rke2/lib/facter/k8s_masters.rb
@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+Facter.add(:k8s_masters) do
+  confine do
+    File.exist?('/etc/rancher/rke2/rke2.yaml') &&
+      File.executable?('/usr/bin/kubectl')
+  end
+
+  setcode do
+    env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' }
+    cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json']
+
+    stdout, stderr, status = Open3.capture3(env, *cmd)
+
+    if status.success?
+      json = JSON.parse(stdout)
+
+      master_count = json['items'].count do |item|
+        roles = item.dig('metadata', 'labels') || {}
+
+        # Look for well-known labels assigned to control-plane nodes
+        roles.any? do |key, _|
+          key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)}
+        end
+      end
+
+      master_count
+    else
+      Facter.debug("kubectl error: #{stderr}")
+      0
+    end
+  rescue StandardError => e
+    Facter.debug("Exception in k8s_masters fact: #{e.message}")
+    0
+  end
+end
--- a/modules/rke2/manifests/config.pp
+++ b/modules/rke2/manifests/config.pp
@ -7,13 +7,6 @@ class rke2::config (
  Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
  String                  $node_token = $rke2::node_token,
  Array[String[1]]        $extra_config_files = $rke2::extra_config_files,
-  Boolean                 $csi_ceph_enable = $rke2::csi_ceph_enable,
-  Array[String]           $csi_ceph_files = $rke2::csi_ceph_files,
-  Array[String]           $csi_ceph_templates = $rke2::csi_ceph_templates,
-  Optional[String[1]]     $csi_ceph_key = $rke2::csi_ceph_key,
-  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::csi_ceph_clusterid,
-  Optional[Array[String]] $csi_ceph_monitors = $rke2::csi_ceph_monitors,
-  Optional[String[1]]     $csi_ceph_poolname = $rke2::csi_ceph_poolname,
 ){

  # if its not the bootstrap node, add join path to config
@ -24,9 +17,7 @@ class rke2::config (
        token  => $node_token,
      } )
    }else{
-      $config = merge($config_hash, {
-        token  => $node_token,
-      } )
+      $config = merge($config_hash, {})
    }
  } elsif $node_type == 'agent' {
      $config = merge($config_hash, {
@ -75,7 +66,7 @@ class rke2::config (
  }

  # on the controller nodes only
-  if $node_type == 'server' {
+  if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 {

    # wait for purelb helm to setup namespace
    if 'purelb' in $facts['k8s_namespaces'] {
@ -114,32 +105,5 @@ class rke2::config (
      }
    }

-    # manage ceph files
-    if $csi_ceph_enable {
-
-      $csi_ceph_files.each |$file| {
-
-        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
-          ensure  => file,
-          owner   => 'root',
-          group   => 'root',
-          mode    => '0644',
-          source  => "puppet:///modules/rke2/${file}.yaml",
-          require => Service['rke2-server'],
-        }
-      }
-
-      $csi_ceph_templates.each |$file| {
-
-        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
-          ensure  => file,
-          owner   => 'root',
-          group   => 'root',
-          mode    => '0644',
-          content => template("rke2/${file}.yaml.erb"),
-          require => Service['rke2-server'],
-        }
-      }
-    }
  }
 }
--- a/modules/rke2/manifests/helm.pp
+++ b/modules/rke2/manifests/helm.pp
@ -20,8 +20,8 @@ class rke2::helm (
      mode   => '0755',
    }

-    # on the controller nodes only
-    if $node_type == 'server' {
+    # on the controller nodes only, and after 3 master nodes exist
+    if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 {

      # check if the repo already exists
      $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {
--- a/modules/rke2/manifests/init.pp
+++ b/modules/rke2/manifests/init.pp
@ -12,13 +12,6 @@ class rke2 (
  Hash                    $helm_repos = $rke2::params::helm_repos,
  Array[String[1]]        $extra_config_files = $rke2::params::extra_config_files,
  Stdlib::HTTPUrl         $container_archive_source = $rke2::params::container_archive_source,
-  Boolean                 $csi_ceph_enable = $rke2::params::csi_ceph_enable,
-  Array[String]           $csi_ceph_files = $rke2::params::csi_ceph_files,
-  Array[String]           $csi_ceph_templates = $rke2::params::csi_ceph_templates,
-  Optional[String[1]]     $csi_ceph_key = $rke2::params::csi_ceph_key,
-  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::params::csi_ceph_clusterid,
-  Optional[Array[String]] $csi_ceph_monitors = $rke2::params::csi_ceph_monitors,
-  Optional[String[1]]     $csi_ceph_poolname = $rke2::params::csi_ceph_poolname,
 ) inherits rke2::params {

  include rke2::install
--- a/modules/rke2/manifests/params.pp
+++ b/modules/rke2/manifests/params.pp
@ -12,11 +12,4 @@ class rke2::params (
  Hash                    $helm_repos = {},
  Array[String[1]]        $extra_config_files = [],
  Stdlib::HTTPUrl         $container_archive_source = 'https://github.com/rancher/rke2/releases/download',
-  Boolean                 $csi_ceph_enable = false,
-  Array[String]           $csi_ceph_files = [],
-  Array[String]           $csi_ceph_templates = [],
-  Optional[String[1]]     $csi_ceph_key = undef,
-  Optional[String[1] ]    $csi_ceph_clusterid = undef,
-  Optional[Array[String]] $csi_ceph_monitors = undef,
-  Optional[String[1]]     $csi_ceph_poolname = undef,
 ) {}
--- a/modules/rke2/templates/ceph-csi-config.yaml.erb
+++ b/modules/rke2/templates/ceph-csi-config.yaml.erb
@ -1,65 +0,0 @@
---
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ceph-csi
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-csi-config
-  namespace: ceph-csi
-data:
-  config.json: |-
-    [
-      {
-        "clusterID": "<%= @csi_ceph_clusterid %>",
-        "monitors": [
-        <% @csi_ceph_monitors.each_with_index do |mon, index| -%>
-          "<%= mon %>"<% if index < @csi_ceph_monitors.length - 1 %>,<% end %>
-        <% end -%>
-        ]
-      }
-    ]
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-csi-encryption-kms-config
-  namespace: ceph-csi
-data:
-  config.json: |-
-    {}
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-config
-  namespace: ceph-csi
-data:
-  ceph.conf: |
-    [global]
-    auth_cluster_required = cephx
-    auth_service_required = cephx
-    auth_client_required = cephx
-  keyring: |
---
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: csi-rbd-sc
-provisioner: rbd.csi.ceph.com
-parameters:
-  clusterID: <%= @csi_ceph_clusterid %>
-  pool: <%= @csi_ceph_poolname %>
-  imageFeatures: layering
-  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi
-  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi
-  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi
-reclaimPolicy: Delete
-allowVolumeExpansion: true
-mountOptions:
-  - discard
--- a/modules/rke2/templates/ceph-csi-secret.yaml.erb
+++ b/modules/rke2/templates/ceph-csi-secret.yaml.erb
@ -1,10 +0,0 @@
---
-apiVersion: v1
-kind: Secret
-metadata:
-  name: csi-rbd-secret
-  namespace: ceph-csi
-stringData:
-  userID: kubernetes
-  userKey: <%= @csi_ceph_key %>
-