feat: remove all ceph configuration from rke2

Remove ceph CSI storage integration from Kubernetes setup.
This removes ceph-related parameters, templates, and RBAC files.

- Remove ceph parameters from rke2 manifests
- Delete ceph CSI template files
- Delete ceph CSI RBAC configuration files
- Remove encrypted ceph key from hieradata

hieradata/roles/infra/k8s/control.eyaml

@@ -1,2 +1 @@
 ---
-rke2::csi_ceph_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEApQ371O4nGSrFB5tOZFTSJP+kJj3wJyEcWiNfonYA5LmbaMnQ6pUortec1519WHMICpSWdpq3O8frivm2CK3taYoKczeTzbsFTxvVp7s6gIZJUsCeqGHuq81YyjPtJE+Yy5IOBJjhe/8ECkEFNr0JlhwKBPWfTx5hHOzRdkGlN464weGFQtCI8UgdGe7AWEePG+u3e4RL+xCriw5tfuqMeeo+isDwVf30nK9NxsnmliOd/+jNW+GrtzycHAeokQOKnxfgrKll5Y5+npy5WueuSCEw1E+Io0NI/4Jthi7zu24UQu0KT8iRsqhuD5mr1ymvCNREnvCcVWt8VVRTGXQV+TBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDS8VXZM3wEAFRALB/Fa19dgDCTRYhU8YY4g9zREP7epY9x2MRWaTT84Jx9w5Dc/XWaRrmL4yL2sK+QHSy2057jHzo=]

hieradata/roles/infra/k8s/control.yaml

@@ -9,23 +9,6 @@ rke2::helm_repos:
   harbor: https://helm.goharbor.io
   traefik: https://traefik.github.io/charts
   hashicorp: https://helm.releases.hashicorp.com
-rke2::csi_ceph_enable: true
-rke2::csi_ceph_clusterid: de96a98f-3d23-465a-a899-86d3d67edab8
-rke2::csi_ceph_poolname: kubernetes
-rke2::csi_ceph_monitors:
-  - 198.18.23.9:6789
-  - 198.18.23.10:6789
-  - 198.18.23.11:6789
-  - 198.18.23.12:6789
-  - 198.18.23.13:6789
-rke2::csi_ceph_files:
-  - ceph-csi-nodeplugin-rbac
-  - ceph-csi-provisioner-rbac
-  - ceph-csi-rbdplugin-provisioner
-  - ceph-csi-rbdplugin
-rke2::csi_ceph_templates:
-  - ceph-csi-config
-  - ceph-csi-secret
 rke2::extra_config_files:
   - rke2-canal-config
   - rke2-nginx-ingress-config

modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml

@@ -1,48 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-nodeplugin
-  namespace: ceph-csi
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["list", "get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-nodeplugin
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-nodeplugin
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-csi-nodeplugin
-  apiGroup: rbac.authorization.k8s.io

modules/rke2/files/ceph-csi-provisioner-rbac.yaml

@@ -1,125 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-csi-provisioner
-  namespace: ceph-csi
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-runner
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update", "patch", "create"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["get", "list", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments/status"]
-    verbs: ["patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationcontents"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationclasses"]
-    verbs: ["get", "list", "watch"]
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: ClusterRole
-  name: rbd-external-provisioner-runner
-  apiGroup: rbac.authorization.k8s.io
-
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-external-provisioner-cfg
-  namespace: ceph-csi
-rules:
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "list", "watch", "create", "update", "delete"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
-
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: rbd-csi-provisioner-role-cfg
-  namespace: ceph-csi
-subjects:
-  - kind: ServiceAccount
-    name: rbd-csi-provisioner
-    namespace: ceph-csi
-roleRef:
-  kind: Role
-  name: rbd-external-provisioner-cfg
-  apiGroup: rbac.authorization.k8s.io

modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml

@@ -1,124 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  selector:
-    app: csi-rbdplugin-provisioner
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: csi-rbdplugin-provisioner
-  namespace: ceph-csi
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: csi-rbdplugin-provisioner
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin-provisioner
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - csi-rbdplugin-provisioner
-              topologyKey: "kubernetes.io/hostname"
-      serviceAccountName: rbd-csi-provisioner
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: csi-rbdplugin
-          image: quay.io/cephcsi/cephcsi:v3.15.0
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--type=rbd"
-            - "--controllerserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--pidlimit=-1"
-            - "--rbdhardmaxclonedepth=8"
-            - "--rbdsoftmaxclonedepth=4"
-            - "--enableprofiling=false"
-            - "--setmetadata=true"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi-provisioner.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-        # snapshotter & other sidecars omitted in this snippet for brevity
-      volumes:
-        - name: socket-dir
-          emptyDir:
-            medium: Memory
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-        # and other volumes as in the original

modules/rke2/files/ceph-csi-rbdplugin.yaml

@@ -1,161 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: csi-rbdplugin
-  namespace: ceph-csi
-spec:
-  selector:
-    matchLabels:
-      app: csi-rbdplugin
-  template:
-    metadata:
-      labels:
-        app: csi-rbdplugin
-    spec:
-      serviceAccountName: rbd-csi-nodeplugin
-      hostNetwork: true
-      hostPID: true
-      priorityClassName: system-node-critical
-      dnsPolicy: ClusterFirstWithHostNet
-      containers:
-        - name: csi-rbdplugin
-          securityContext:
-            privileged: true
-          capabilities:
-            add: ["SYS_ADMIN"]
-          allowPrivilegeEscalation: true
-          image: quay.io/cephcsi/cephcsi:v3.15.0
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--pluginpath=/var/lib/kubelet/plugins"
-            - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/"
-            - "--type=rbd"
-            - "--nodeserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=rbd.csi.ceph.com"
-            - "--enableprofiling=false"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-            - name: CSI_ADDONS_ENDPOINT
-              value: unix:///csi/csi-addons.sock
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: host-dev
-              mountPath: /dev
-            - name: host-sys
-              mountPath: /sys
-            - name: host-mount
-              mountPath: /run/mount
-            - name: etc-selinux
-              mountPath: /etc/selinux
-              readOnly: true
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: plugin-dir
-              mountPath: /var/lib/kubelet/plugins
-              mountPropagation: "Bidirectional"
-            - name: mountpoint-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-            - name: ceph-logdir
-              mountPath: /var/log/ceph
-            - name: ceph-config
-              mountPath: /etc/ceph/
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: ceph-csi-encryption-kms-config
-              mountPath: /etc/ceph-csi-encryption-kms-config/
-            - name: oidc-token
-              mountPath: /run/secrets/tokens
-              readOnly: true
-        # possibly sidecars like driver-registrar, liveness, etc.
-      volumes:
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
-            type: DirectoryOrCreate
-        - name: plugin-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins
-            type: Directory
-        - name: mountpoint-dir
-          hostPath:
-            path: /var/lib/kubelet/pods
-            type: DirectoryOrCreate
-        - name: ceph-logdir
-          hostPath:
-            path: /var/log/ceph
-            type: DirectoryOrCreate
-        - name: host-dev
-          hostPath:
-            path: /dev
-        - name: host-sys
-          hostPath:
-            path: /sys
-        - name: etc-selinux
-          hostPath:
-            path: /etc/selinux
-            type: DirectoryOrCreate
-        - name: host-mount
-          hostPath:
-            path: /run/mount
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-            type: DirectoryOrCreate
-        - name: ceph-config
-          configMap:
-            name: ceph-config
-        - name: ceph-csi-config
-          configMap:
-            name: ceph-csi-config
-        - name: ceph-csi-encryption-kms-config
-          configMap:
-            name: ceph-csi-encryption-kms-config
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: Memory
-        - name: oidc-token
-          projected:
-            sources:
-              - serviceAccountToken:
-                path: oidc-token
-                expirationSeconds: 3607
-                audience: ceph-csi-kms
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-metrics-rbdplugin
-  namespace: ceph-csi
-  labels:
-    app: csi-metrics
-spec:
-  ports:
-    - name: http-metrics
-      port: 8080
-      protocol: TCP
-      targetPort: 8680
-  selector:
-    app: csi-rbdplugin

modules/rke2/manifests/config.pp

@@ -7,13 +7,6 @@ class rke2::config (
   Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
   String                  $node_token = $rke2::node_token,
   Array[String[1]]        $extra_config_files = $rke2::extra_config_files,
-  Boolean                 $csi_ceph_enable = $rke2::csi_ceph_enable,
-  Array[String]           $csi_ceph_files = $rke2::csi_ceph_files,
-  Array[String]           $csi_ceph_templates = $rke2::csi_ceph_templates,
-  Optional[String[1]]     $csi_ceph_key = $rke2::csi_ceph_key,
-  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::csi_ceph_clusterid,
-  Optional[Array[String]] $csi_ceph_monitors = $rke2::csi_ceph_monitors,
-  Optional[String[1]]     $csi_ceph_poolname = $rke2::csi_ceph_poolname,
 ){
 
   # if its not the bootstrap node, add join path to config
@@ -112,32 +105,5 @@ class rke2::config (
       }
     }
 
-    # manage ceph files
-    if $csi_ceph_enable {
-
-      $csi_ceph_files.each |$file| {
-
-        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
-          ensure  => file,
-          owner   => 'root',
-          group   => 'root',
-          mode    => '0644',
-          source  => "puppet:///modules/rke2/${file}.yaml",
-          require => Service['rke2-server'],
-        }
-      }
-
-      $csi_ceph_templates.each |$file| {
-
-        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
-          ensure  => file,
-          owner   => 'root',
-          group   => 'root',
-          mode    => '0644',
-          content => template("rke2/${file}.yaml.erb"),
-          require => Service['rke2-server'],
-        }
-      }
-    }
   }
 }

modules/rke2/manifests/init.pp

@@ -12,13 +12,6 @@ class rke2 (
   Hash                    $helm_repos = $rke2::params::helm_repos,
   Array[String[1]]        $extra_config_files = $rke2::params::extra_config_files,
   Stdlib::HTTPUrl         $container_archive_source = $rke2::params::container_archive_source,
-  Boolean                 $csi_ceph_enable = $rke2::params::csi_ceph_enable,
-  Array[String]           $csi_ceph_files = $rke2::params::csi_ceph_files,
-  Array[String]           $csi_ceph_templates = $rke2::params::csi_ceph_templates,
-  Optional[String[1]]     $csi_ceph_key = $rke2::params::csi_ceph_key,
-  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::params::csi_ceph_clusterid,
-  Optional[Array[String]] $csi_ceph_monitors = $rke2::params::csi_ceph_monitors,
-  Optional[String[1]]     $csi_ceph_poolname = $rke2::params::csi_ceph_poolname,
 ) inherits rke2::params {
 
   include rke2::install

modules/rke2/manifests/params.pp

@@ -12,11 +12,4 @@ class rke2::params (
   Hash                    $helm_repos = {},
   Array[String[1]]        $extra_config_files = [],
   Stdlib::HTTPUrl         $container_archive_source = 'https://github.com/rancher/rke2/releases/download',
-  Boolean                 $csi_ceph_enable = false,
-  Array[String]           $csi_ceph_files = [],
-  Array[String]           $csi_ceph_templates = [],
-  Optional[String[1]]     $csi_ceph_key = undef,
-  Optional[String[1] ]    $csi_ceph_clusterid = undef,
-  Optional[Array[String]] $csi_ceph_monitors = undef,
-  Optional[String[1]]     $csi_ceph_poolname = undef,
 ) {}

modules/rke2/templates/ceph-csi-config.yaml.erb

@@ -1,65 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ceph-csi
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-csi-config
-  namespace: ceph-csi
-data:
-  config.json: |-
-    [
-      {
-        "clusterID": "<%= @csi_ceph_clusterid %>",
-        "monitors": [
-        <% @csi_ceph_monitors.each_with_index do |mon, index| -%>
-          "<%= mon %>"<% if index < @csi_ceph_monitors.length - 1 %>,<% end %>
-        <% end -%>
-        ]
-      }
-    ]
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-csi-encryption-kms-config
-  namespace: ceph-csi
-data:
-  config.json: |-
-    {}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ceph-config
-  namespace: ceph-csi
-data:
-  ceph.conf: |
-    [global]
-    auth_cluster_required = cephx
-    auth_service_required = cephx
-    auth_client_required = cephx
-  keyring: |
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: csi-rbd-sc
-provisioner: rbd.csi.ceph.com
-parameters:
-  clusterID: <%= @csi_ceph_clusterid %>
-  pool: <%= @csi_ceph_poolname %>
-  imageFeatures: layering
-  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi
-  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi
-  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
-  csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi
-reclaimPolicy: Delete
-allowVolumeExpansion: true
-mountOptions:
-  - discard

modules/rke2/templates/ceph-csi-secret.yaml.erb

@@ -1,10 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: csi-rbd-secret
-  namespace: ceph-csi
-stringData:
-  userID: kubernetes
-  userKey: <%= @csi_ceph_key %>
-