feat: moderate the k8s install (#403)

- only install a base config
- wait for 3 masters before deploying helm charts
- remove cluster-domain
- manage nginx ingres via rke2 helmconfig

Reviewed-on: #403

modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml

@@ -1,48 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-nodeplugin
-  namespace: ceph-csi
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["list", "get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-nodeplugin
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-csi-nodeplugin
-  apiGroup: rbac.authorization.k8s.io

modules/rke2/files/ceph-csi-provisioner-rbac.yaml

@@ -1,125 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-provisioner
-  namespace: ceph-csi
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-runner
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update", "patch", "create"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["get", "list", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments/status"]
-    verbs: ["patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationcontents"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationclasses"]
-    verbs: ["get", "list", "watch"]
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-external-provisioner-runner
-  apiGroup: rbac.authorization.k8s.io
-
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-cfg
-  namespace: ceph-csi
-rules:
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "list", "watch", "create", "update", "delete"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
-
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role-cfg
-  namespace: ceph-csi
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: Role
-  name: rbd-external-provisioner-cfg
-  apiGroup: rbac.authorization.k8s.io

modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml

@@ -1,124 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  selector:
-    app: csi-rbdplugin-provisioner
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: csi-rbdplugin-provisioner
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin-provisioner
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - csi-rbdplugin-provisioner
-              topologyKey: "kubernetes.io/hostname"
-      serviceAccountName: rbd-csi-provisioner
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: csi-rbdplugin
-          image: quay.io/cephcsi/cephcsi:v3.15
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--type=rbd"
-            - "--controllerserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--pidlimit=-1"
-            - "--rbdhardmaxclonedepth=8"
-            - "--rbdsoftmaxclonedepth=4"
-            - "--enableprofiling=false"
-            - "--setmetadata=true"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi-provisioner.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-        # snapshotter & other sidecars omitted in this snippet for brevity
-      volumes:
-        - name: socket-dir
-          emptyDir:
-            medium: Memory
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-        # and other volumes as in the original

modules/rke2/files/ceph-csi-rbdplugin.yaml

@@ -1,155 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: csi-rbdplugin
-  namespace: ceph-csi
-spec:
-  selector:
-    matchLabels:
-      app: csi-rbdplugin
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin
-    spec:
-      serviceAccountName: rbd-csi-nodeplugin
-      hostNetwork: true
-      hostPID: true
-      priorityClassName: system-node-critical
-      dnsPolicy: ClusterFirstWithHostNet
-      containers:
-        - name: csi-rbdplugin
-          securityContext:
-            privileged: true
-          capabilities:
-            add: ["SYS_ADMIN"]
-          allowPrivilegeEscalation: true
-          image: quay.io/cephcsi/cephcsi:v3.15
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--pluginpath=/var/lib/kubelet/plugins"
-            - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/"
-            - "--type=rbd"
-            - "--nodeserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--enableprofiling=false"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: host-mount
-              mountPath: /run/mount
-            - name: etc-selinux
-              mountPath: /etc/selinux
-              readOnly: true
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: plugin-dir
-              mountPath: /var/lib/kubelet/plugins
-              mountPropagation: "Bidirectional"
-            - name: mountpoint-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-            - name: ceph-logdir
-              mountPath: /var/log/ceph
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: oidc-token
-              mountPath: /run/secrets/tokens
-              readOnly: true
-        # possibly sidecars like driver-registrar, liveness, etc.
-      volumes:
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
-            type: DirectoryOrCreate
-        - name: plugin-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins
-            type: Directory
-        - name: mountpoint-dir
-          hostPath:
-            path: /var/lib/kubelet/pods
-            type: DirectoryOrCreate
-        - name: ceph-logdir
-          hostPath:
-            path: /var/log/ceph
-            type: DirectoryOrCreate
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: etc-selinux
-          hostPath:
-            path: /etc/selinux
-            type: DirectoryOrCreate
-        - name: host-mount
-          hostPath:
-            path: /run/mount
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-            type: DirectoryOrCreate
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-metrics-rbdplugin
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-  selector:
-    app: csi-rbdplugin

modules/rke2/files/ingress-route-rancher.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

modules/rke2/files/rke2-nginx-ingress-config.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      hostPort:
+        enabled: false
+      service:
+        enabled: true
+        type: LoadBalancer
+        externalTrafficPolicy: Local
+        loadBalancerClass: purelb.io/purelb
+        allocateLoadBalancerNodePorts: false
+        annotations:
+          purelb.io/service-group: common
+          purelb.io/addresses: "198.18.200.0"

modules/rke2/files/service-loadbalancer-nginx.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: rke2-ingress-nginx-controller
-  namespace: kube-system
-  annotations:
-    purelb.io/service-group: common
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Cluster
-  ports:
-    - name: http
-      port: 80
-      targetPort: http
-      protocol: TCP
-    - name: https
-      port: 443
-      targetPort: https
-      protocol: TCP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/instance: rke2-ingress-nginx
-    loadBalancerIP: 198.18.200.0
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChartConfig
-metadata:
-  name: rke2-ingress-nginx
-  namespace: kube-system
-spec:
-  valuesContent: |-
-    controller:
-      hostPort:
-        enabled: false
-      service:
-        enabled: true
-        type: LoadBalancer
-        externalTrafficPolicy: Local
-        annotations:
-          purelb.io/service-group: common