feat: deploy k8s

- import k8s role
- add worker/controller roles
- add worker/controller profiles

Puppetfile

@@ -43,6 +43,7 @@ mod 'puppet-letsencrypt', '11.0.0'
 mod 'puppet-rundeck', '9.1.0'
 mod 'puppet-redis', '11.0.0'
 mod 'puppet-nodejs', '11.0.0'
+mod 'puppet-k8s', '2.0.1'
 
 # other
 mod 'ghoneycutt-puppet', '3.3.0'

hieradata/common.yaml

@@ -135,6 +135,8 @@ lookup_options:
   keepalived::vrrp_instance:
     merge:
       strategy: deep
+  k8s::server::resources::bootstrap::secret:
+    convert_to: "Sensitive"
 
 facts_path: '/opt/puppetlabs/facter/facts.d'
 

hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml

@@ -4,4 +4,4 @@ networking::interfaces:
     ipaddress: 198.18.13.77
 networking::routes:
   default:
-    gateway: 198.18.13.254
+    gateway: 198.18.13.254

hieradata/roles/infra/k8s.eyaml

@@ -0,0 +1,2 @@
+---
+k8s::server::resources::bootstrap::secret: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEALdIVlhinA6Cuv4fGPIFBxNKyGGfdaJ5o/xGVgdm3I/7M+PmxMFkl7GwE1G0jqsvJD61NNMrONA9aFUyW8aYHYKE9EfbxgH80RJFxWmFuJjPf1N78hrguuOOa5nc92waiCatdyTyqgIuqEmoIxnTAFikMSv2dAHKfMlUWLkl9LO3XQEzLiDbZfCNJHdnh1ACU04TXdeNFci2xfBk9xz0pdnKMQzKKrsyR005E5i+gc9mTFD2Gujde1y0ahiqnJC7sCPOGseY+myDJuaBFxJRBDEcDlI+XgpyVteawAJEYxjHP9aTWGpwTwkUI1QJgzz7tNQA5AdRzrYI3C3/uiXG4sjBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDHmeAhkAJ4L74kUAm6yzFegCApiPyJeMe61xgVZAwP57Bh/767LGnr6rg5S9k3p21xkQ==]

hieradata/roles/infra/k8s.yaml

@@ -0,0 +1,11 @@
+k8s::node::manage_simple_cni: false
+k8s::server::resources::manage_flannel: false
+
+k8s::container_manager: containerd
+k8s::version: 1.28.14
+k8s::etcd_version: 3.5.16
+
+### k8s::install::crictl
+k8s::install::crictl::config:
+  'runtime-endpoint': 'unix:///run/containerd/containerd.sock'
+  'image-endpoint': 'unix:///run/containerd/containerd.sock'

hieradata/roles/infra/k8s/controller.yaml

@@ -0,0 +1,44 @@
+---
+hiera_include:
+  - profiles::k8s::controller
+
+### K8S::Server
+k8s::server::node_on_server: false
+k8s::server::manage_kubeadm: true
+k8s::server::etcd::generate_ca: true
+#k8s::server::etcd::client_ca_cert: '/etc/pki/tls/vault/certificate.crt'
+#k8s::server::etcd::client_ca_key: '/etc/pki/tls/vault/private.key'
+
+### K8S::Server::Apiserver
+# Choose an interface which is for cluster communications.
+# The apiserver will expose a port on the controller
+# and all the workers need to be able to reach it.
+k8s::server::apiserver::advertise_address: "%{facts.networking.ip}"
+
+### K8S::Server::Resources
+k8s::server::resources::manage_flannel: false
+
+consul::services:
+  k8s:
+    service_name: 'k8s'
+    tags:
+      - 'containers'
+      - 'k8s'
+      - 'kubernetes'
+    address: "%{facts.networking.ip}"
+    port: 6443
+    checks:
+      - id: 'k8s_tcp_check'
+        name: 'K8S TCP Check'
+        tcp: "%{facts.networking.fqdn}:6443"
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: k8s
+    disposition: write
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - k8s.service.consul
+  - k8s.query.consul

hieradata/roles/infra/k8s/worker.yaml

@@ -0,0 +1,7 @@
+---
+hiera_include:
+  - profiles::k8s::worker
+
+### K8S::Node
+k8s::node::node_token: "puppet.%{lookup('k8s::server::resources::bootstrap::secret')}"
+k8s::node::manage_crictl: true

site/profiles/manifests/k8s/controller.pp

@@ -0,0 +1,37 @@
+# Class: profile::k8s::controller
+#
+# @param container_manager set the cri, like cri-o or containerd, if controller should be also a worker
+# @param etcd_version version of etcd
+# @param k8s_version version of kubernetes
+# @param manage_firewall whether to manage firewall or not
+# @param manage_kube_proxy whether to manage manage_kube_proxy or not
+# @param control_plane_url
+#   api server url where the server/nodes connect to.
+#   this is most likely a load balanced dns with all the controllers in the backend.
+#   on single head clusters this may be the dns name:port of the controller node.
+# @param role role in the cluster, server, node, none
+# @param puppetdb_discovery whether to use puppetdb or not
+# @param service_cidr address space for the services
+# @param pod_cidr address space for the pods
+#
+# lint:ignore:autoloader_layout
+class profiles::k8s::controller (
+  # lint:endignore
+  Boolean $manage_firewall                   = false,                     # k8s-class default: false
+  Boolean $manage_kube_proxy                 = true,                      # k8s-class default: true
+  Boolean $puppetdb_discovery                = true,                      # k8s-class default: false
+  Stdlib::HTTPUrl $control_plane_url         = 'https://k8s.service.consul:6443', # k8s-class default: https://kubernetes:6443
+  Enum['server'] $role                       = 'server',                  # k8s-class default: none
+  K8s::CIDR $service_cidr                    = '10.20.0.0/20',            # k8s-class default: 10.1.0.0/24
+  K8s::CIDR $pod_cidr                        = '10.20.16.0/20',           # k8s-class default: 10.0.0.0/16
+) {
+  class { 'k8s':
+    manage_firewall      => $manage_firewall,
+    manage_kube_proxy    => $manage_kube_proxy,
+    control_plane_url    => $control_plane_url,
+    role                 => $role,
+    service_cluster_cidr => $service_cidr,
+    cluster_cidr         => $pod_cidr,
+    puppetdb_discovery   => $puppetdb_discovery,
+  }
+}

site/profiles/manifests/k8s/worker.pp

@@ -0,0 +1,30 @@
+# Class: profile::k8s::worker
+#
+# @param role role in the cluster, server, node, none
+# @param control_plane_url
+#   cluster url where the server/nodes connect to.
+#   this is most likely a load balanced dns with all the controllers in the backend.
+#   on single head clusters this may be the dns name:port of the controller node.
+# @param k8s_version version of kubernetes
+# @param puppetdb_discovery whether to use puppetdb or not
+# @param manage_firewall whether to manage firewall or not
+# @param manage_kube_proxy whether to manage manage_kube_proxy or not, for cilium this is not needed
+# @param container_manager set the cri, like cri-o or containerd
+#
+# lint:ignore:autoloader_layout
+class profiles::k8s::worker (
+  # lint:endignore
+  Boolean $manage_firewall                   = true,         # k8s-class default: false
+  Boolean $manage_kube_proxy                 = true,         # k8s-class default: true
+  Boolean $puppetdb_discovery                = true,         # k8s-class default: false
+  Enum['node'] $role                         = 'node',       # k8s-class default: none
+  Stdlib::HTTPUrl $control_plane_url         = $profiles::k8s::controller::control_plane_url,
+) {
+  class { 'k8s':
+    manage_firewall    => $manage_firewall,
+    manage_kube_proxy  => $manage_kube_proxy,
+    control_plane_url  => $control_plane_url,
+    puppetdb_discovery => $puppetdb_discovery,
+    role               => $role,
+  }
+}

site/roles/manifests/infra/k8s/controller.pp

@@ -0,0 +1,11 @@
+# deploy k8s controllers
+class roles::infra::k8s::controller {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

site/roles/manifests/infra/k8s/worker.pp

@@ -0,0 +1,11 @@
+# deploy k8s workers
+class roles::infra::k8s::worker {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}