From efbbb6bcb132beecfbeb46be6f7eb55b541f9e55 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sun, 12 Oct 2025 17:50:24 +1100 Subject: [PATCH] feat: moderate the k8s install (#403) - only install a base config - wait for 3 masters before deploying helm charts - remove cluster-domain - manage nginx ingres via rke2 helmconfig Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/403 --- hieradata/roles/infra/k8s.eyaml | 2 +- hieradata/roles/infra/k8s/control.eyaml | 1 - hieradata/roles/infra/k8s/control.yaml | 20 +-- .../rke2/files/ceph-csi-nodeplugin-rbac.yaml | 48 ------ .../rke2/files/ceph-csi-provisioner-rbac.yaml | 125 -------------- .../files/ceph-csi-rbdplugin-provisioner.yaml | 124 -------------- modules/rke2/files/ceph-csi-rbdplugin.yaml | 155 ------------------ modules/rke2/files/ingress-route-rancher.yaml | 1 + .../rke2/files/rke2-nginx-ingress-config.yaml | 20 +++ .../files/service-loadbalancer-nginx.yaml | 41 ----- modules/rke2/lib/facter/k8s_masters.rb | 39 +++++ modules/rke2/manifests/config.pp | 40 +---- modules/rke2/manifests/helm.pp | 4 +- modules/rke2/manifests/init.pp | 7 - modules/rke2/manifests/params.pp | 7 - .../rke2/templates/ceph-csi-config.yaml.erb | 65 -------- .../rke2/templates/ceph-csi-secret.yaml.erb | 10 -- 17 files changed, 66 insertions(+), 643 deletions(-) delete mode 100644 modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml delete mode 100644 modules/rke2/files/ceph-csi-provisioner-rbac.yaml delete mode 100644 modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml delete mode 100644 modules/rke2/files/ceph-csi-rbdplugin.yaml create mode 100644 modules/rke2/files/rke2-nginx-ingress-config.yaml delete mode 100644 modules/rke2/files/service-loadbalancer-nginx.yaml create mode 100644 modules/rke2/lib/facter/k8s_masters.rb delete mode 100644 modules/rke2/templates/ceph-csi-config.yaml.erb delete mode 100644 modules/rke2/templates/ceph-csi-secret.yaml.erb diff --git a/hieradata/roles/infra/k8s.eyaml b/hieradata/roles/infra/k8s.eyaml index 56f2d4b..78340d6 100644 --- a/hieradata/roles/infra/k8s.eyaml +++ b/hieradata/roles/infra/k8s.eyaml @@ -1 +1 @@ -rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAPRBcsDJv/SDR4pOi4fBWzU2ayCXW8BaATzpg3h8mHsVPcHwhfL9w9d4BIshM5yzSZwfmj8EBMk6sa4vEcBRN5r5vWDETnjwsvtLULvyiPm5WVYAYe+Z/fzoCUqswWvXrU/yzoMd/oUYFOrtCB0wT4bZX2leg64CQ7DZIf4wZod0Z9EH7YwUpL4mmbLWY29w0Z3F702O8v0hp9J7IWX5A5ob8OLaTy9SmApZEtm35UDz+OAaJ1OZ9yc2pWTkVZ0dSgMvouynJWToZ/k0ZwoW5I32WLeZ6mkYODwXF1ULqVTX3eX/c9HlVilHJar9TlcY46+8ypV3Uoju8j6c86DaRtjCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQe3h1bWn1G2HYEwupc7n3EoBwzcA4ICV4efMd1n4d1CW+wlBAoa5T8lXSluf09vajUQmaFhl3pl99v3uc5M38ePPd5QkzoZPfb4kLcmuufGNfwbIXpaQQ+nwQATOX1UG5SOEovshvWnW0iF5JrgIa+MLed9WF48NMbvZZLbwyBe7hrQ==] +rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAOD+w5nJFqEYWFj+tZQ65Oi19eDhaWtpLQ0gwEdBtMmY9sPJ63l1q2qH933NH6TOd1UlMDvGfoDLCae+yt/MAW//cJ15X3QbiVQ23DdfCOlUEZN6fjVrveEt/yIeFQmWvnkMS4pRfPbgQu2OHm37PpuPE7s6dUyGItAYjchrRhtQ7ibhXDnN7miG+oVRXP2T8b/V5WPdmA222DSV6r/AnqaWkna9W/oh/I2sNKeEm5q3f8bh8Gxt1dDy3VwaZ3lAh3uR+SUm7P/6PTYw8opxiumFBvos0mRiXIdOwUuqrAS8hafWBhxnDLlTBfz62Nc4wQmQ8gz0bHJZSipH9G6mIEDCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ2WG6ROWFlQdXx0TuO5oABoBwTDtAXIj7y6I1B3zCnFoMHETf5d7ulPGdgwZsENf0UIHpg2l0w503MUHHbu6YDFiDiTE0oDNJPVHid7TO+XwWFgh5v1MWi/XeEBgCs6nMCW8qkX0Z3UXaZdSBUll1M4sRtuqscBnoD/LLs2kKfxrqQg==] diff --git a/hieradata/roles/infra/k8s/control.eyaml b/hieradata/roles/infra/k8s/control.eyaml index 50b5073..ed97d53 100644 --- a/hieradata/roles/infra/k8s/control.eyaml +++ b/hieradata/roles/infra/k8s/control.eyaml @@ -1,2 +1 @@ --- -rke2::csi_ceph_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEApQ371O4nGSrFB5tOZFTSJP+kJj3wJyEcWiNfonYA5LmbaMnQ6pUortec1519WHMICpSWdpq3O8frivm2CK3taYoKczeTzbsFTxvVp7s6gIZJUsCeqGHuq81YyjPtJE+Yy5IOBJjhe/8ECkEFNr0JlhwKBPWfTx5hHOzRdkGlN464weGFQtCI8UgdGe7AWEePG+u3e4RL+xCriw5tfuqMeeo+isDwVf30nK9NxsnmliOd/+jNW+GrtzycHAeokQOKnxfgrKll5Y5+npy5WueuSCEw1E+Io0NI/4Jthi7zu24UQu0KT8iRsqhuD5mr1ymvCNREnvCcVWt8VVRTGXQV+TBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDS8VXZM3wEAFRALB/Fa19dgDCTRYhU8YY4g9zREP7epY9x2MRWaTT84Jx9w5Dc/XWaRrmL4yL2sK+QHSy2057jHzo=] diff --git a/hieradata/roles/infra/k8s/control.yaml b/hieradata/roles/infra/k8s/control.yaml index 28a130e..28dc8a0 100644 --- a/hieradata/roles/infra/k8s/control.yaml +++ b/hieradata/roles/infra/k8s/control.yaml @@ -9,29 +9,11 @@ rke2::helm_repos: harbor: https://helm.goharbor.io traefik: https://traefik.github.io/charts hashicorp: https://helm.releases.hashicorp.com -rke2::csi_ceph_enable: true -rke2::csi_ceph_clusterid: de96a98f-3d23-465a-a899-86d3d67edab8 -rke2::csi_ceph_poolname: kubernetes -rke2::csi_ceph_monitors: - - 198.18.23.9:6789 - - 198.18.23.10:6789 - - 198.18.23.11:6789 - - 198.18.23.12:6789 - - 198.18.23.13:6789 -rke2::csi_ceph_files: - - ceph-csi-nodeplugin-rbac - - ceph-csi-provisioner-rbac - - ceph-csi-rbdplugin-provisioner - - ceph-csi-rbdplugin -rke2::csi_ceph_templates: - - ceph-csi-config - - ceph-csi-secret rke2::extra_config_files: - rke2-canal-config - - service-loadbalancer-nginx + - rke2-nginx-ingress-config rke2::config_hash: advertise-address: "%{hiera('networking_loopback0_ip')}" - cluster-domain: "svc.k8s.unkin.net" tls-san: - "join-k8s.service.consul" - "api-k8s.service.consul" diff --git a/modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml b/modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml deleted file mode 100644 index 8be3689..0000000 --- a/modules/rke2/files/ceph-csi-nodeplugin-rbac.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rbd-csi-nodeplugin - namespace: ceph-csi - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["list", "get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rbd-csi-nodeplugin - namespace: ceph-csi -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io diff --git a/modules/rke2/files/ceph-csi-provisioner-rbac.yaml b/modules/rke2/files/ceph-csi-provisioner-rbac.yaml deleted file mode 100644 index 36ed47b..0000000 --- a/modules/rke2/files/ceph-csi-provisioner-rbac.yaml +++ /dev/null @@ -1,125 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rbd-csi-provisioner - namespace: ceph-csi - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update", "patch", "create"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["get", "list", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationcontents"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationclasses"] - verbs: ["get", "list", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rbd-csi-provisioner - namespace: ceph-csi -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-cfg - namespace: ceph-csi -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: ceph-csi -subjects: - - kind: ServiceAccount - name: rbd-csi-provisioner - namespace: ceph-csi -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io diff --git a/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml b/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml deleted file mode 100644 index 8bd065f..0000000 --- a/modules/rke2/files/ceph-csi-rbdplugin-provisioner.yaml +++ /dev/null @@ -1,124 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: csi-rbdplugin-provisioner - namespace: ceph-csi - labels: - app: csi-metrics -spec: - selector: - app: csi-rbdplugin-provisioner - ports: - - name: http-metrics - port: 8080 - protocol: TCP - targetPort: 8680 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: csi-rbdplugin-provisioner - namespace: ceph-csi -spec: - replicas: 3 - selector: - matchLabels: - app: csi-rbdplugin-provisioner - template: - metadata: - labels: - app: csi-rbdplugin-provisioner - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-rbdplugin-provisioner - topologyKey: "kubernetes.io/hostname" - serviceAccountName: rbd-csi-provisioner - priorityClassName: system-cluster-critical - containers: - - name: csi-rbdplugin - image: quay.io/cephcsi/cephcsi:v3.15 - args: - - "--nodeid=$(NODE_ID)" - - "--type=rbd" - - "--controllerserver=true" - - "--endpoint=$(CSI_ENDPOINT)" - - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" - - "--v=5" - - "--drivername=rbd.csi.ceph.com" - - "--pidlimit=-1" - - "--rbdhardmaxclonedepth=8" - - "--rbdsoftmaxclonedepth=4" - - "--enableprofiling=false" - - "--setmetadata=true" - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - - name: CSI_ADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - imagePullPolicy: IfNotPresent - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: host-dev - mountPath: /dev - - name: host-sys - mountPath: /sys - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: ceph-csi-encryption-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - - name: ceph-config - mountPath: /etc/ceph/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - # snapshotter & other sidecars omitted in this snippet for brevity - volumes: - - name: socket-dir - emptyDir: - medium: Memory - - name: host-dev - hostPath: - path: /dev - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: ceph-csi-config - configMap: - name: ceph-csi-config - - name: ceph-csi-encryption-kms-config - configMap: - name: ceph-csi-encryption-kms-config - - name: ceph-config - configMap: - name: ceph-config - - name: keys-tmp-dir - emptyDir: - medium: Memory - # and other volumes as in the original diff --git a/modules/rke2/files/ceph-csi-rbdplugin.yaml b/modules/rke2/files/ceph-csi-rbdplugin.yaml deleted file mode 100644 index 3e09c87..0000000 --- a/modules/rke2/files/ceph-csi-rbdplugin.yaml +++ /dev/null @@ -1,155 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-rbdplugin - namespace: ceph-csi -spec: - selector: - matchLabels: - app: csi-rbdplugin - template: - metadata: - labels: - app: csi-rbdplugin - spec: - serviceAccountName: rbd-csi-nodeplugin - hostNetwork: true - hostPID: true - priorityClassName: system-node-critical - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: csi-rbdplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: quay.io/cephcsi/cephcsi:v3.15 - args: - - "--nodeid=$(NODE_ID)" - - "--pluginpath=/var/lib/kubelet/plugins" - - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/" - - "--type=rbd" - - "--nodeserver=true" - - "--endpoint=$(CSI_ENDPOINT)" - - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" - - "--v=5" - - "--drivername=rbd.csi.ceph.com" - - "--enableprofiling=false" - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: CSI_ADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - imagePullPolicy: IfNotPresent - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: host-dev - mountPath: /dev - - name: host-sys - mountPath: /sys - - name: host-mount - mountPath: /run/mount - - name: etc-selinux - mountPath: /etc/selinux - readOnly: true - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins - mountPropagation: "Bidirectional" - - name: mountpoint-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: "Bidirectional" - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - - name: ceph-logdir - mountPath: /var/log/ceph - - name: ceph-config - mountPath: /etc/ceph/ - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: ceph-csi-encryption-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - - name: oidc-token - mountPath: /run/secrets/tokens - readOnly: true - # possibly sidecars like driver-registrar, liveness, etc. - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/rbd.csi.ceph.com - type: DirectoryOrCreate - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins - type: Directory - - name: mountpoint-dir - hostPath: - path: /var/lib/kubelet/pods - type: DirectoryOrCreate - - name: ceph-logdir - hostPath: - path: /var/log/ceph - type: DirectoryOrCreate - - name: host-dev - hostPath: - path: /dev - - name: host-sys - hostPath: - path: /sys - - name: etc-selinux - hostPath: - path: /etc/selinux - type: DirectoryOrCreate - - name: host-mount - hostPath: - path: /run/mount - - name: lib-modules - hostPath: - path: /lib/modules - type: DirectoryOrCreate - - name: ceph-config - configMap: - name: ceph-config - - name: ceph-csi-config - configMap: - name: ceph-csi-config - - name: ceph-csi-encryption-kms-config - configMap: - name: ceph-csi-encryption-kms-config - - name: keys-tmp-dir - emptyDir: - medium: Memory - ---- -apiVersion: v1 -kind: Service -metadata: - name: csi-metrics-rbdplugin - namespace: ceph-csi - labels: - app: csi-metrics -spec: - ports: - - name: http-metrics - port: 8080 - protocol: TCP - targetPort: 8680 - selector: - app: csi-rbdplugin diff --git a/modules/rke2/files/ingress-route-rancher.yaml b/modules/rke2/files/ingress-route-rancher.yaml index 1797c17..eb189f3 100644 --- a/modules/rke2/files/ingress-route-rancher.yaml +++ b/modules/rke2/files/ingress-route-rancher.yaml @@ -1,3 +1,4 @@ +--- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/modules/rke2/files/rke2-nginx-ingress-config.yaml b/modules/rke2/files/rke2-nginx-ingress-config.yaml new file mode 100644 index 0000000..c51271e --- /dev/null +++ b/modules/rke2/files/rke2-nginx-ingress-config.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-ingress-nginx + namespace: kube-system +spec: + valuesContent: |- + controller: + hostPort: + enabled: false + service: + enabled: true + type: LoadBalancer + externalTrafficPolicy: Local + loadBalancerClass: purelb.io/purelb + allocateLoadBalancerNodePorts: false + annotations: + purelb.io/service-group: common + purelb.io/addresses: "198.18.200.0" diff --git a/modules/rke2/files/service-loadbalancer-nginx.yaml b/modules/rke2/files/service-loadbalancer-nginx.yaml deleted file mode 100644 index 4615dc1..0000000 --- a/modules/rke2/files/service-loadbalancer-nginx.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rke2-ingress-nginx-controller - namespace: kube-system - annotations: - purelb.io/service-group: common -spec: - type: LoadBalancer - externalTrafficPolicy: Cluster - ports: - - name: http - port: 80 - targetPort: http - protocol: TCP - - name: https - port: 443 - targetPort: https - protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/component: controller - app.kubernetes.io/instance: rke2-ingress-nginx - loadBalancerIP: 198.18.200.0 ---- -apiVersion: helm.cattle.io/v1 -kind: HelmChartConfig -metadata: - name: rke2-ingress-nginx - namespace: kube-system -spec: - valuesContent: |- - controller: - hostPort: - enabled: false - service: - enabled: true - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - purelb.io/service-group: common diff --git a/modules/rke2/lib/facter/k8s_masters.rb b/modules/rke2/lib/facter/k8s_masters.rb new file mode 100644 index 0000000..33b9c84 --- /dev/null +++ b/modules/rke2/lib/facter/k8s_masters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'json' +require 'open3' + +Facter.add(:k8s_masters) do + confine do + File.exist?('/etc/rancher/rke2/rke2.yaml') && + File.executable?('/usr/bin/kubectl') + end + + setcode do + env = { 'KUBECONFIG' => '/etc/rancher/rke2/rke2.yaml' } + cmd = ['/usr/bin/kubectl', 'get', 'nodes', '-o', 'json'] + + stdout, stderr, status = Open3.capture3(env, *cmd) + + if status.success? + json = JSON.parse(stdout) + + master_count = json['items'].count do |item| + roles = item.dig('metadata', 'labels') || {} + + # Look for well-known labels assigned to control-plane nodes + roles.any? do |key, _| + key =~ %r{node-role\.kubernetes\.io/(control-plane|master|etcd)} + end + end + + master_count + else + Facter.debug("kubectl error: #{stderr}") + 0 + end + rescue StandardError => e + Facter.debug("Exception in k8s_masters fact: #{e.message}") + 0 + end +end diff --git a/modules/rke2/manifests/config.pp b/modules/rke2/manifests/config.pp index e4affb7..72d4c70 100644 --- a/modules/rke2/manifests/config.pp +++ b/modules/rke2/manifests/config.pp @@ -7,13 +7,6 @@ class rke2::config ( Stdlib::Fqdn $bootstrap_node = $rke2::bootstrap_node, String $node_token = $rke2::node_token, Array[String[1]] $extra_config_files = $rke2::extra_config_files, - Boolean $csi_ceph_enable = $rke2::csi_ceph_enable, - Array[String] $csi_ceph_files = $rke2::csi_ceph_files, - Array[String] $csi_ceph_templates = $rke2::csi_ceph_templates, - Optional[String[1]] $csi_ceph_key = $rke2::csi_ceph_key, - Optional[String[1] ] $csi_ceph_clusterid = $rke2::csi_ceph_clusterid, - Optional[Array[String]] $csi_ceph_monitors = $rke2::csi_ceph_monitors, - Optional[String[1]] $csi_ceph_poolname = $rke2::csi_ceph_poolname, ){ # if its not the bootstrap node, add join path to config @@ -24,9 +17,7 @@ class rke2::config ( token => $node_token, } ) }else{ - $config = merge($config_hash, { - token => $node_token, - } ) + $config = merge($config_hash, {}) } } elsif $node_type == 'agent' { $config = merge($config_hash, { @@ -75,7 +66,7 @@ class rke2::config ( } # on the controller nodes only - if $node_type == 'server' { + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 { # wait for purelb helm to setup namespace if 'purelb' in $facts['k8s_namespaces'] { @@ -114,32 +105,5 @@ class rke2::config ( } } - # manage ceph files - if $csi_ceph_enable { - - $csi_ceph_files.each |$file| { - - file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml": - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => "puppet:///modules/rke2/${file}.yaml", - require => Service['rke2-server'], - } - } - - $csi_ceph_templates.each |$file| { - - file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml": - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - content => template("rke2/${file}.yaml.erb"), - require => Service['rke2-server'], - } - } - } } } diff --git a/modules/rke2/manifests/helm.pp b/modules/rke2/manifests/helm.pp index e172f2e..8eb5536 100644 --- a/modules/rke2/manifests/helm.pp +++ b/modules/rke2/manifests/helm.pp @@ -20,8 +20,8 @@ class rke2::helm ( mode => '0755', } - # on the controller nodes only - if $node_type == 'server' { + # on the controller nodes only, and after 3 master nodes exist + if $node_type == 'server' and $facts['k8s_masters'] and $facts['k8s_masters'] > 2 { # check if the repo already exists $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | { diff --git a/modules/rke2/manifests/init.pp b/modules/rke2/manifests/init.pp index 86f7e67..882eee6 100644 --- a/modules/rke2/manifests/init.pp +++ b/modules/rke2/manifests/init.pp @@ -12,13 +12,6 @@ class rke2 ( Hash $helm_repos = $rke2::params::helm_repos, Array[String[1]] $extra_config_files = $rke2::params::extra_config_files, Stdlib::HTTPUrl $container_archive_source = $rke2::params::container_archive_source, - Boolean $csi_ceph_enable = $rke2::params::csi_ceph_enable, - Array[String] $csi_ceph_files = $rke2::params::csi_ceph_files, - Array[String] $csi_ceph_templates = $rke2::params::csi_ceph_templates, - Optional[String[1]] $csi_ceph_key = $rke2::params::csi_ceph_key, - Optional[String[1] ] $csi_ceph_clusterid = $rke2::params::csi_ceph_clusterid, - Optional[Array[String]] $csi_ceph_monitors = $rke2::params::csi_ceph_monitors, - Optional[String[1]] $csi_ceph_poolname = $rke2::params::csi_ceph_poolname, ) inherits rke2::params { include rke2::install diff --git a/modules/rke2/manifests/params.pp b/modules/rke2/manifests/params.pp index 3aa4951..0689852 100644 --- a/modules/rke2/manifests/params.pp +++ b/modules/rke2/manifests/params.pp @@ -12,11 +12,4 @@ class rke2::params ( Hash $helm_repos = {}, Array[String[1]] $extra_config_files = [], Stdlib::HTTPUrl $container_archive_source = 'https://github.com/rancher/rke2/releases/download', - Boolean $csi_ceph_enable = false, - Array[String] $csi_ceph_files = [], - Array[String] $csi_ceph_templates = [], - Optional[String[1]] $csi_ceph_key = undef, - Optional[String[1] ] $csi_ceph_clusterid = undef, - Optional[Array[String]] $csi_ceph_monitors = undef, - Optional[String[1]] $csi_ceph_poolname = undef, ) {} diff --git a/modules/rke2/templates/ceph-csi-config.yaml.erb b/modules/rke2/templates/ceph-csi-config.yaml.erb deleted file mode 100644 index cf5b484..0000000 --- a/modules/rke2/templates/ceph-csi-config.yaml.erb +++ /dev/null @@ -1,65 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ceph-csi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ceph-csi-config - namespace: ceph-csi -data: - config.json: |- - [ - { - "clusterID": "<%= @csi_ceph_clusterid %>", - "monitors": [ - <% @csi_ceph_monitors.each_with_index do |mon, index| -%> - "<%= mon %>"<% if index < @csi_ceph_monitors.length - 1 %>,<% end %> - <% end -%> - ] - } - ] ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ceph-csi-encryption-kms-config - namespace: ceph-csi -data: - config.json: |- - {} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ceph-config - namespace: ceph-csi -data: - ceph.conf: | - [global] - auth_cluster_required = cephx - auth_service_required = cephx - auth_client_required = cephx - keyring: | ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-rbd-sc -provisioner: rbd.csi.ceph.com -parameters: - clusterID: <%= @csi_ceph_clusterid %> - pool: <%= @csi_ceph_poolname %> - imageFeatures: layering - csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret - csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi - csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret - csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi - csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret - csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi -reclaimPolicy: Delete -allowVolumeExpansion: true -mountOptions: - - discard diff --git a/modules/rke2/templates/ceph-csi-secret.yaml.erb b/modules/rke2/templates/ceph-csi-secret.yaml.erb deleted file mode 100644 index 7141198..0000000 --- a/modules/rke2/templates/ceph-csi-secret.yaml.erb +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-rbd-secret - namespace: ceph-csi -stringData: - userID: kubernetes - userKey: <%= @csi_ceph_key %> -