feat: add ectd role for k8s

- small updates to k8s server/node/etcd-replica
- change k8s networking to all use networkd

Puppetfile

@@ -43,6 +43,7 @@ mod 'puppet-letsencrypt', '11.1.0'
 mod 'puppet-rundeck', '9.2.0'
 mod 'puppet-redis', '11.1.0'
 mod 'puppet-nodejs', '11.0.0'
+mod 'puppet-k8s', '2.0.1'
 
 # other
 mod 'saz-sudo', '9.0.2'

hieradata/common.yaml

@@ -141,6 +141,8 @@ lookup_options:
   keepalived::vrrp_instance:
     merge:
       strategy: deep
+  k8s::server::resources::bootstrap::secret:
+    convert_to: "Sensitive"
   profiles::etcd::node::initial_cluster_token:
     convert_to: Sensitive
   sysctl::base::values:

hieradata/roles/infra/dns/master.yaml

@@ -17,6 +17,7 @@ profiles::dns::master::acls:
       - 198.18.27.0/24
       - 198.18.28.0/24
       - 198.18.29.0/24
+      - 198.18.30.0/24
 
 profiles::dns::master::zones:
   main.unkin.net:

hieradata/roles/infra/dns/resolver.yaml

@@ -26,6 +26,7 @@ profiles::dns::resolver::acls:
       - 198.18.27.0/24
       - 198.18.28.0/24
       - 198.18.29.0/24
+      - 198.18.30.0/24
   acl-nomad-jobs:
     addresses:
       - 198.18.64.0/24

hieradata/roles/infra/etcd/k8s.eyaml

@@ -0,0 +1,2 @@
+---
+profiles::etcd::node::initial_cluster_token: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAhLyXszXUU6Dkiw9bEJTH0RXGaV2751NzvLH94i7QHfNukvOslF/kaDOA+FwqG06xSKSKo24Qyj4ewYA3BzhN8XLf2E9uW2LuDrUoA6aXUP2tYPqiTw8zmmgsVV5t7Y5PeNcleV3KmfcJZJKp33yGCKtGF7ggvNvnied5slO6E1BDkcVnqO7sdyI0MqSvsvH4IvEmeiSWAcBRBnwVLIwfn10frIvUg0fH4uZR7DASfO/HstYWKAEacz4xYBv74TtVVtYHlPvnVwC20YIYDMrgBsm3XngyWIQvruQCgyIkRzHjUKCpp76HpyEqzdJdEdaywkODYNOT6ab1B5uUu9WaMjBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBADXLPOqFHdnVgJW5+iXJYcgCDK1Eyr+RwvMA+3VszYALU5B6OCH5maplwC5aUgiQZ7ew==]

hieradata/roles/infra/etcd/k8s.yaml

@@ -0,0 +1,64 @@
+---
+hiera_include:
+  - profiles::etcd::node
+
+profiles::etcd::node::members_lookup: true
+profiles::etcd::node::members_role: roles::infra::etcd::k8s
+
+profiles::etcd::node::config:
+  data-dir: /data/etcd
+  client-cert-auth: false
+  client-transport-security:
+    cert-file: /etc/pki/tls/vault/certificate.crt
+    key-file: /etc/pki/tls/vault/private.key
+    client-cert-auth: false
+    auto-tls: false
+  peer-transport-security:
+    cert-file: /etc/pki/tls/vault/certificate.crt
+    key-file: /etc/pki/tls/vault/private.key
+    client-cert-auth: false
+    auto-tls: false
+  allowed-cn:
+  max-wals: 5
+  max-snapshots: 5
+  snapshot-count: 10000
+  heartbeat-interval: 100
+  election-timeout: 1000
+  cipher-suites: [
+    TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+    TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+  ]
+  tls-min-version: 'TLS1.2'
+  tls-max-version: 'TLS1.3'
+
+profiles::pki::vault::alt_names:
+  - etcd-k8s.service.consul
+  - etcd-k8s.query.consul
+  - "etcd-k8s.service.%{facts.country}-%{facts.region}.consul"
+
+profiles::ssh::sign::principals:
+  - etcd-k8s.query.consul
+  - etcd-k8s.service.consul
+  - etcd-k8s.service.%{facts.country}-%{facts.region}.consul
+
+consul::services:
+  etcd:
+    service_name: 'etcd-k8s'
+    tags:
+      - 'etcd'
+      - 'k8s'
+      - 'etcd-k8s'
+    address: "%{facts.networking.ip}"
+    port: 2379
+    checks:
+      - id: 'etcd_http_health_check'
+        name: 'ETCD HTTP Health Check'
+        http: "https://%{facts.networking.ip}:2379/health"
+        method: 'GET'
+        interval: '10s'
+        timeout: '1s'
+        tls_skip_verify: true
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: etcd-k8s
+    disposition: write

hieradata/roles/infra/k8s.eyaml

@@ -0,0 +1,2 @@
+---
+k8s::server::resources::bootstrap::secret: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEALdIVlhinA6Cuv4fGPIFBxNKyGGfdaJ5o/xGVgdm3I/7M+PmxMFkl7GwE1G0jqsvJD61NNMrONA9aFUyW8aYHYKE9EfbxgH80RJFxWmFuJjPf1N78hrguuOOa5nc92waiCatdyTyqgIuqEmoIxnTAFikMSv2dAHKfMlUWLkl9LO3XQEzLiDbZfCNJHdnh1ACU04TXdeNFci2xfBk9xz0pdnKMQzKKrsyR005E5i+gc9mTFD2Gujde1y0ahiqnJC7sCPOGseY+myDJuaBFxJRBDEcDlI+XgpyVteawAJEYxjHP9aTWGpwTwkUI1QJgzz7tNQA5AdRzrYI3C3/uiXG4sjBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDHmeAhkAJ4L74kUAm6yzFegCApiPyJeMe61xgVZAwP57Bh/767LGnr6rg5S9k3p21xkQ==]

hieradata/roles/infra/k8s.yaml

@@ -0,0 +1,30 @@
+---
+hiera_include:
+  - k8s
+
+k8s::node::manage_simple_cni: false
+k8s::server::resources::manage_flannel: false
+
+k8s::container_manager: containerd
+k8s::version: 1.31.9
+k8s::etcd_version: 3.5.16
+k8s::cluster_domain: k8s.au-syd1.unkin.net
+k8s::manage_firewall: false
+k8s::manage_kube_proxy: true
+k8s::puppetdb_discovery: true
+k8s::service_cidr: 198.18.30.0/24
+k8s::pod_cidr: 10.240.0.0/16
+
+### k8s::install::crictl
+k8s::install::crictl::config:
+  'runtime-endpoint': 'unix:///run/containerd/containerd.sock'
+  'image-endpoint': 'unix:///run/containerd/containerd.sock'
+
+### networking
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true

hieradata/roles/infra/k8s/controller.yaml

@@ -0,0 +1,30 @@
+---
+k8s::role: server
+k8s::server::node_on_server: false
+k8s::server::manage_kubeadm: true
+k8s::server::apiserver::advertise_address: "%{facts.networking.ip}"
+
+consul::services:
+  k8s:
+    service_name: 'k8s'
+    tags:
+      - 'containers'
+      - 'k8s'
+      - 'kubernetes'
+    address: "%{facts.networking.ip}"
+    port: 6443
+    checks:
+      - id: 'k8s_tcp_check'
+        name: 'K8S TCP Check'
+        tcp: "%{facts.networking.fqdn}:6443"
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: k8s
+    disposition: write
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - k8s.service.consul
+  - k8s.query.consul

hieradata/roles/infra/k8s/etcd.yaml

@@ -0,0 +1,7 @@
+---
+hiera_include:
+  - k8s::server::etcd
+k8s::role: none
+k8s::server::etcd::generate_ca: true
+k8s::server::etcd::manage_members: true
+k8s::server::etcd::self_signed_tls: true

hieradata/roles/infra/k8s/worker.yaml

@@ -0,0 +1,5 @@
+---
+### K8S::Node
+k8s::role: node
+k8s::node::node_token: "puppet.%{lookup('k8s::server::resources::bootstrap::secret')}"
+k8s::node::manage_crictl: true

modules/libs/lib/facter/subnet_facts.rb

@@ -21,7 +21,8 @@ class SubnetAttributes
     '198.18.26.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0010
     '198.18.27.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0011
     '198.18.28.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0012
-    '198.18.29.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }  # common node0013
+    '198.18.29.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0013
+    '198.18.30.0/24' => { environment: 'prod', region: 'syd1', country: 'au' } # k8s service network
   }.freeze
 
   # Default attributes if no subnet matches, also defined as a constant

site/roles/manifests/infra/etcd/k8s.pp

@@ -0,0 +1,11 @@
+# a role to deploy etcd for k8s
+class roles::infra::etcd::k8s {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

site/roles/manifests/infra/k8s/controller.pp

@@ -0,0 +1,11 @@
+# deploy k8s controllers
+class roles::infra::k8s::controller {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

site/roles/manifests/infra/k8s/etcd.pp

@@ -0,0 +1,11 @@
+# deploy k8s workers
+class roles::infra::k8s::etcd {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

site/roles/manifests/infra/k8s/worker.pp

@@ -0,0 +1,11 @@
+# deploy k8s workers
+class roles::infra::k8s::worker {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}