feat: adding rke2

- manage rke2 repos
- add rke2 module (init, params, install, config, service)
- exclude setting ips for cilium interfaces
- split roles::infra::k8s::node -> control/compute roles
- add bootstrap_node, manage server and token fields in rke2 config

hieradata/common.yaml

@@ -368,6 +368,28 @@ profiles::ceph::client::mons:
   - 10.18.15.1
   - 10.18.15.2
   - 10.18.15.3
+
+profiles::ceph::conf::config:
+  global:
+    auth_client_required: 'cephx'
+    auth_cluster_required: 'cephx'
+    auth_service_required: 'cephx'
+    fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+    mon_allow_pool_delete: true
+    mon_initial_members: 'prodnxsr0009,prodnxsr0010,prodnxsr0011,prodnxsr0012,prodnxsr0013'
+    mon_host: '198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13'
+    ms_bind_ipv4: true
+    ms_bind_ipv6: false
+    osd_crush_chooseleaf_type: 1
+    osd_pool_default_min_size: 2
+    osd_pool_default_size: 3
+    osd_pool_default_pg_num: 128
+    public_network: >
+      198.18.23.1/32,198.18.23.2/32,198.18.23.3/32,198.18.23.4/32,
+      198.18.23.5/32,198.18.23.6/32,198.18.23.7/32,198.18.23.8/32,
+      198.18.23.9/32,198.18.23.10/32,198.18.23.11/32,198.18.23.12/32,
+      198.18.23.13/32
+
 #profiles::base::hosts::additional_hosts:
 #  - ip: 198.18.17.9
 #    hostname: prodinf01n09.main.unkin.net

hieradata/country/au/region/syd1/infra/halb/haproxy2.yaml

@@ -10,6 +10,7 @@ profiles::haproxy::dns::vrrp_cnames:
   - git.unkin.net
   - fafflix.unkin.net
   - grafana.unkin.net
+  - dashboard.ceph.unkin.net
 
 profiles::haproxy::mappings:
   fe_http:
@@ -27,6 +28,7 @@ profiles::haproxy::mappings:
       - 'fafflix.unkin.net be_jellyfin'
       - 'git.unkin.net be_gitea'
       - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'
   fe_https:
     ensure: present
     mappings:
@@ -42,6 +44,7 @@ profiles::haproxy::mappings:
       - 'fafflix.unkin.net be_jellyfin'
       - 'git.unkin.net be_gitea'
       - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'
 
 profiles::haproxy::frontends:
   fe_http:
@@ -62,6 +65,7 @@ profiles::haproxy::frontends:
         - 'acl_fafflix req.hdr(host) -i fafflix.unkin.net'
         - 'acl_gitea req.hdr(host) -i git.unkin.net'
         - 'acl_grafana req.hdr(host) -i grafana.unkin.net'
+        - 'acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net'
         - 'acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24'
       use_backend:
         - "%[req.hdr(host),lower,map(/etc/haproxy/fe_https.map,be_default)]"
@@ -79,6 +83,7 @@ profiles::haproxy::frontends:
         - 'set-header X-Frame-Options DENY if acl_fafflix'
         - 'set-header X-Frame-Options DENY if acl_gitea'
         - 'set-header X-Frame-Options DENY if acl_grafana'
+        - 'set-header X-Frame-Options DENY if acl_ceph_dashboard'
         - 'set-header X-Content-Type-Options nosniff'
         - 'set-header X-XSS-Protection 1;mode=block'
 
@@ -262,6 +267,26 @@ profiles::haproxy::backends:
       redirect: 'scheme https if !{ ssl_fc }'
       stick-table: 'type ip size 200k expire 30m'
       stick: 'on src'
+  be_ceph_dashboard:
+    description: Backend for Ceph Dashboard from Mgr instances
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-check:
+        - expect status 200
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 9443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+      stick-table: 'type ip size 200k expire 30m'
+      stick: 'on src'
 
 profiles::haproxy::certlist::enabled: true
 profiles::haproxy::certlist::certificates:
@@ -276,6 +301,7 @@ profiles::haproxy::certlist::certificates:
   - /etc/pki/tls/letsencrypt/fafflix.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/letsencrypt/git.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/letsencrypt/grafana.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/dashboard.ceph.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/vault/certificate.pem
 
 # additional altnames
@@ -303,3 +329,4 @@ certbot::client::domains:
   - fafflix.unkin.net
   - git.unkin.net
   - grafana.unkin.net
+  - dashboard.ceph.unkin.net

hieradata/roles/apps/media/prowlarr.yaml

@@ -70,7 +70,7 @@ profiles::nginx::simpleproxy::locations:
   arrstack_web_external:
     location_satisfy: any
     location_allow:
-      - 198.18.13.47
+      - 198.18.26.161
-      - 198.18.13.50
+      - 198.18.27.131
-      - 198.18.13.51
+      - 198.18.28.165
-      - 198.18.13.52
+      - 198.18.29.32

hieradata/roles/infra/auth/glauth.yaml

@@ -211,6 +211,18 @@ glauth::users:
     loginshell: '/bin/bash'
     homedir: '/home/waewak'
     passsha256: 'd9bb99634215fe031c3bdca94149a165192fe8384ecaa238a19354c2f760a811'
+  debvin:
+    user_name: 'debvin'
+    givenname: 'Debbie'
+    sn: 'Vincent'
+    mail: 'debvin@users.main.unkin.net'
+    uidnumber: 20009
+    primarygroup: 20000
+    othergroups:
+      - 20010 # jelly
+    loginshell: '/bin/bash'
+    homedir: '/home/debvin'
+    passsha256: 'cdac05ddb02e665d4ea65a974995f38a10236bc158731d92d78f6cde89b294a1'
 
 glauth::services:
   svc_jellyfin:

hieradata/roles/infra/ceph/rgw.eyaml

@@ -0,0 +1,8 @@
+---
+
+profiles::ceph::rgw::ceph_client_keys:
+  ausyd1nxvm2115: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAovOUUtTm/9nXWj6D+kVhmbZedVVkA5N80ULNYqISrv0A+KlXCx/V2sB56SpJ9eL6aPHdC+CKkrtrrdo1h2j2em7wTA5ghhbFVF0NIS7bbn0jzpl5YqbUrNxOtDSpjzX1aCGnyMvw69Lp+NcHwxIj+1XFgK6335138s9wbX3HmYF3jcsQkMqTzynELe1OQPWFXVKTjUfFFLdCQOFryp8UY8L9j/PpV6wd4w6p7R6eXhX21rjSaN4aqN1zjsnF2OVhL8Ge0QxMhePWKGOqsUfi72kh3II028DjqU0DcZQvoxnoqRPyUUjysH0nTKoLeXOGNgJdphY1dHBJ+SJnw2gqQDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB9wA3ZJjmU95W+9r8va5uagDAMAv2APfdzrPnAU7NJPL+IW08osGuQWWamqF+XGVeHRoBmoFKwZ7grYRV2e3aabyc=]
+  ausyd1nxvm2116: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAf5ksy/pyUZSwTh+HiKw+1Uhj16A0DVZEKAbkKUQzXVmc+QpL4Dn7YBoXlEwrY8CcsTrxTQjADvtu9FC3o34QIdh06noSgYYA+7fna2A+9+oYoNtgwC3b8LeglxO/SQ9dKoJ90jRtmlw5P/CtrxA2RelMK6FNRekp1CaWMM4q20fJGgr/E33vgx38UJyp4/q0bTu2lLehCuDUP80j3XGbSNZ2snfYdIo91Cl+nSxLSU2TdnFpWaabsH19HwDnkWGiILlLBVvvhY7copCxs5DS1ueoOTCsqnWSrTrBMJjnu7WZd/s4NLw/0q/UP5xcFA51caY3Kv+sI6bfIYkNoLazwDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDrCF16nrtukRPamx1VbGY+gDAK5dw0kV8MpATpwxTkJG6JtlFlwdpU9THs1bNwqSRD1ZhEWxQeWwsyyTtjUXi4bP8=]
+  ausyd1nxvm2117: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAKtvsgDk2QTmL8flBBE4nA43sRSnroq4I6T2CUAYv/lRdzCCWvE961o/51zyQEz8L5QyDo7L3gcGYIBqthYjRe9Gp5a5d4qds0qskupgQKnb0KR2RLFVUtH5vxHqyJZHjXaP+PQreyRoSIfRWXAdoZu544FeJ9DKKbmEzZaH5B2OdDMrf4Ufuud0maEAw0PJthS//ghCfGi74F1xlJnIWVvMhp66b0iMxC+ACClEHunG3oKx7M/w05HllG0wcxPTg4PFrbnFXjRuIxsykF9aVHJkRnCdgbMXRM4o6FrYyZRR74F1HKRujFCUA7kYWDKLxHxJpYCvCHp4HMhfzjs824zBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBbaSig6kgVVNfmSI53sNimgDDQ5O10Dzfa7S7RdJVLLUFBaZ5TG2g3Bwmy0k3wKZvABYMuYyOxQdfk6eMsKC+sC5w=]
+  ausyd1nxvm2118: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAdKl0ude3ZxW0ihyA758mpQp43qZvzTI2Bp83WzCp2ifJCgAIjRWdd01P64rSHaa4lus/wqB9jxg38g6DrN4ejX56Y/CJ6GQKxz6b1BO5nDfsLx6QEzCt+cfg5d/PPoTtEpz2VSvDfxFUrHiABA6++Sqzb9Og+nQCFMYJD3NHCk67QpkjPGQ/ejZk4MNXZQVCfKOlFqay/fF0jEmQixFOlX/Fdm9UoKttbrKluUmzpaVUzfGRaTTFVgzc3x2t/z1q1k0P7ClI9Uu02kUXpFVs9LPX99Zc2GtrnP06mYqqARhWF1NMK0zlmxtKpfObahRP/HmtI3fgnQsU1Cpwah0emTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAILqpYx3FKY3xXLJRu2oDlgDCIOXeX6hxpu0qpj5c/9jMUSeV2DIydnxO+MiT3mceS50ip8B+zGQy5UedPmLt36Zs=]
+  ausyd1nxvm2119: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEASlZglUxazp+9azfV3QkgRv+ACo+MH0RO5b18blbelgdmr38iwK7MwwEFpEfVJEyc/ph9RunWwrMmofDQHj5bBribfzZ2pH2CGiOrR0i5lZMtN0yQXPBA/+jm1Pi1AWGJLtoquuhMbibuHOTiXwBCBVrHHHaFTR5Xt34ABN/p/mCaG+N9nWux93msHCCextCalKBMmPhmI2q6HodfjanEVgYAe3/5hRPnpsi6IGSDNGygsTC3MG+hjGMpNF8izbwk9Lpzn6kY51aeNxI2ed9Jm8UZ/k+8b+o7ZQyWIBbf7DTFpEzk4G46puaDbXIorBWQ4azCjN3gt8VB91hwihtzcDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB6U+9z4cSzMTA1z9bmoX82gDBfy5zbRPK8GxImJo6evecMOTtaY2c4aEnESXtGBCS02enmxljv9dv1UYQD0/a6S3A=]

hieradata/roles/infra/ceph/rgw.yaml

@@ -0,0 +1,60 @@
+---
+hiera_include:
+  - profiles::ceph::rgw
+  - profiles::nginx::simpleproxy
+
+profiles::ceph::rgw::enable: true
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - radosgw.main.unkin.net
+  - radosgw.service.consul
+  - radosgw.query.consul
+  - "radosgw.service.%{facts.country}-%{facts.region}.consul"
+
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+
+# manage a simple nginx reverse proxy
+profiles::nginx::simpleproxy::nginx_vhost: 'radosgw.service.consul'
+profiles::nginx::simpleproxy::nginx_aliases:
+  - radosgw.service.au-syd1.consul
+profiles::nginx::simpleproxy::proxy_port: 7480
+profiles::nginx::simpleproxy::proxy_path: '/'
+nginx::client_max_body_size: 100M
+
+# manage consul service
+consul::services:
+  radosgw:
+    service_name: 'radosgw'
+    address: "%{facts.networking.ip}"
+    port: 443
+    checks:
+      - id: 'radosgw_https_check'
+        name: 'RADOSGW HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: radosgw
+    disposition: write

hieradata/roles/infra/incus/node.yaml

@@ -6,6 +6,7 @@ hiera_include:
   - zfs
   - profiles::ceph::node
   - profiles::ceph::client
+  - profiles::ceph::dashboard
   - profiles::storage::cephfsvols
   - exporters::frr_exporter
 
@@ -36,6 +37,7 @@ profiles::ssh::sign::principals:
   - "%{facts.networking.interfaces.enp3s0.ip}"
 
 # configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
 consul::services:
   incus:
     service_name: 'incus'
@@ -121,6 +123,9 @@ profiles::yum::global::repos:
 # dns
 profiles::dns::base::primary_interface: loopback0
 
+# dashboard/haproxy
+profiles::ceph::dashboard::ipaddress: "%{hiera('networking_loopback0_ip')}"
+
 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true

hieradata/roles/infra/k8s/control.eyaml

@@ -0,0 +1 @@
+rke2::node_token: ENC[PKCS7,MIIBuQYJKoZIhvcNAQcDoIIBqjCCAaYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWU56ILL1Te14z0vG9Q9tD9DOlRfxasZHuChZlWHiw3MLOv3FwEc9+jEW20zMJCw6xoTITJ6mXN1fXe48fT3hX5jaBijoqLpfWp64zXomXCEyLh+DZn1N8ETRFS/gB6Fk1PjkCv4x4jd92tTI28gQQAJCb5oNTJzeCFJtovECSL2ElpRJKvdkf+fFtTHdpaTcX4oASz2WUyVux8oAwkGUCRlulzpAJhcwCYoKVAq1wiFRaicK/vZ9DJBlN1HrixKNCsKwRYLZYg0Qh0NjjJPI3UlrR6l06qnTvHhYAOg2BvfQIVlcUI5NjOfUD96GqTb8hFoWhsfhQ0GQT385QmsMlDB8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCj5SiNg/nNkDbK8gR+45sMgFCzjCs/vVqJ+dFQA7r7nVhejb/bKiRtWmEnnvDSN0zOVHAueCeImNBfWB3+ZDSPbXvjN6audmdmeMV+9rvq05Nxcsb5Lcawh6DUzkJWazGH3A==]

hieradata/roles/infra/k8s/control.yaml

@@ -1,42 +1,203 @@
 ---
-profiles::pki::vault::alt_names:
+hiera_include:
-  - k8s-control.service.consul
+  - profiles::selinux::setenforce
-  - k8s-control.query.consul
+  - frrouting
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - exporters::frr_exporter
+  - rke2
 
-profiles::ssh::sign::principals:
+# manage rke2
-  - k8s-control.service.consul
+rke2::bootstrap_node: prodnxsr0001.main.unkin.net
-  - k8s-control.query.consul
+rke2::join_url: https://join-k8s.service.consul:9345
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
+rke2::node_type: server
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "join-k8s.service.consul"
+    - "api-k8s.service.consul"
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: cilium
 
-# configure consul service
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
-consul::services:
+python::manage_dev_package: false
-  k8s-control:
+
-    service_name: 'k8s-control'
+profiles::packages::include:
-    tags:
+  bridge-utils: {}
-      - 'k8s'
+  cephadm: {}
-      - 'container'
+
-    address: "%{facts.networking.fqdn}"
+profiles::selinux::setenforce::mode: disabled
-    port: 6443
+
-    checks:
+profiles::ceph::client::manage_ceph_conf: false
-      - id: 'k8s-control_https_check'
+profiles::ceph::client::manage_ceph_package: false
-        name: 'k8s-control HTTPS Check'
+profiles::ceph::client::manage_ceph_paths: false
-        http: "https://%{facts.networking.fqdn}:6443"
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
-        method: 'GET'
+profiles::ceph::client::mons:
-        tls_skip_verify: true
+  - 198.18.23.9
-        interval: '10s'
+  - 198.18.23.10
-        timeout: '1s'
+  - 198.18.23.11
-profiles::consul::client::node_rules:
+  - 198.18.23.12
-  - resource: service
+  - 198.18.23.13
-    segment: k8s-control
+
-    disposition: write
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+
+# dns
+profiles::dns::base::primary_interface: loopback0
 
 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true
 networking::interfaces:
-  eth0:
+  "%{hiera('networking_1000_iface')}":
     type: physical
+    ipaddress: "%{hiera('networking_1000_ip')}"
+    gateway: 198.18.15.254
+    txqueuelen: 10000
     forwarding: true
-    dhcp: true
+  "%{hiera('networking_2500_iface')}":
+    type: physical
+    ipaddress: "%{hiera('networking_2500_ip')}"
     mtu: 1500
+    txqueuelen: 10000
+    forwarding: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback1:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback1_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback2:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback2_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+consul::services:
+  api-k8s:
+    service_name: 'api-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 6443
+    checks:
+      - id: 'api-k8s_livez_check'
+        name: 'api-k8s livez Check'
+        args:
+          - sudo
+          - /usr/local/bin/check_k8s_api.sh
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'rke2_tcp_check_9345'
+        name: 'rke2 TCP Check 9345'
+        tcp: "%{hiera('networking_loopback0_ip')}:9345"
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
+    disposition: write
+  - resource: service
+    segment: frr_exporter
+    disposition: write
+
+# frrouting
+exporters::frr_exporter::enable: true
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  "%{hiera('networking_1000_iface')}":
+    area: 0.0.0.0
+  "%{hiera('networking_2500_iface')}":
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+  loopback1:
+    area: 0.0.0.0
+  loopback2:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# add loopback interfaces to ssh list
+ssh::server::options:
+  ListenAddress:
+    - "%{hiera('networking_loopback0_ip')}"
+    - "%{hiera('networking_1000_ip')}"
+    - "%{hiera('networking_2500_ip')}"
+
+profiles::ssh::sign::principals:
+  - "%{hiera('networking_loopback0_ip')}"
+  - "%{hiera('networking_1000_ip')}"
+  - "%{hiera('networking_2500_ip')}"
+
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"
+
+sudo::configs:
+  consul-checks:
+    priority: 20
+    content: |
+      consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh

hieradata/roles/infra/k8s/node.yaml

@@ -5,6 +5,24 @@ hiera_include:
   - profiles::ceph::node
   - profiles::ceph::client
   - exporters::frr_exporter
+  - profiles::rke2::node
+
+# manage rke2
+profiles::rke2::node::servers:
+  - prodnxsr0001.main.unkin.net
+  - prodnxsr0002.main.unkin.net
+  - prodnxsr0003.main.unkin.net
+
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: cilium
 
 # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
 python::manage_dev_package: false
@@ -25,6 +43,7 @@ profiles::ceph::client::mons:
   - 198.18.23.11
   - 198.18.23.12
   - 198.18.23.13
+
 # additional repos
 profiles::yum::global::repos:
   ceph:
@@ -55,6 +74,20 @@ profiles::yum::global::repos:
     baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
     gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
     mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
 
 # dns
 profiles::dns::base::primary_interface: loopback0
@@ -91,8 +124,37 @@ networking::interfaces:
     netmask: 255.255.255.255
     mtu: 1500
 
-# consul
+# configure consul service
+consul::services:
+  api-k8s:
+    service_name: 'api-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 6443
+    checks:
+      - id: 'api-k8s_https_check'
+        name: 'api-k8s HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:6443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'etcd_tcp_check_9345'
+        name: 'ETCD TCP Check 9345'
+        tcp: "%{facts.networking.fqdn}:9345"
+        interval: '10s'
+        timeout: '1s'
 profiles::consul::client::node_rules:
+  - resource: service
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
+    disposition: write
   - resource: service
     segment: frr_exporter
     disposition: write
@@ -127,3 +189,8 @@ profiles::ssh::sign::principals:
   - "%{hiera('networking_loopback0_ip')}"
   - "%{hiera('networking_1000_ip')}"
   - "%{hiera('networking_2500_ip')}"
+
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"

hieradata/roles/infra/metrics/grafana.yaml

@@ -11,6 +11,9 @@ profiles::metrics::grafana::db_name: "%{hiera('profiles::sql::postgresdb::dbname
 profiles::metrics::grafana::db_user: "%{hiera('profiles::sql::postgresdb::dbuser')}"
 profiles::metrics::grafana::db_pass: "%{hiera('profiles::sql::postgresdb::dbpass')}"
 profiles::metrics::grafana::pgsql_backend: true
+profiles::metrics::grafana::plugins:
+  victoriametrics-logs-datasource:
+    ensure: present
 
 # additional altnames
 profiles::pki::vault::alt_names:

hieradata/roles/infra/pki/certbot.yaml

@@ -15,3 +15,4 @@ certbot::domains:
   - fafflix.unkin.net
   - git.unkin.net
   - grafana.unkin.net
+  - dashboard.ceph.unkin.net

modules/rke2/files/check_k8s_api.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/bash
+/var/lib/rancher/rke2/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml get --raw /livez

modules/rke2/manifests/config.pp

@@ -0,0 +1,49 @@
+# config rke2
+class rke2::config (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+  Hash                    $config_hash = $rke2::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
+  String                  $node_token = $rke2::node_token,
+){
+
+  # if its not the bootstrap node, add join path to config
+  if $node_type == 'server' {
+    if $trusted['certname'] != $bootstrap_node {
+      $config = merge($config_hash, {
+        server => $join_url,
+        token  => $node_token,
+      } )
+    }else{
+      $config = $config_hash
+    }
+  }else{
+    $config = $config_hash
+  }
+
+  # create the config file
+  file { $config_file:
+    ensure  => file,
+    content => Sensitive($config.to_yaml),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }
+
+  # create a script to verify k8s api is up (used by consul)
+  file {'/usr/local/bin/check_k8s_api.sh':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/rke2/check_k8s_api.sh'
+  }
+
+  # symlink kubectl to path
+  file {'/usr/bin/kubectl':
+    ensure => link,
+    target => '/var/lib/rancher/rke2/bin/kubectl',
+  }
+
+}

modules/rke2/manifests/init.pp

@@ -0,0 +1,16 @@
+# manage rke2
+class rke2 (
+  Enum['server', 'agent'] $node_type = $rke2::params::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::params::config_file,
+  Hash                    $config_hash = $rke2::params::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::params::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::params::bootstrap_node,
+  String                  $node_token = $rke2::params::node_token,
+) inherits rke2::params {
+
+  include rke2::install
+  include rke2::config
+  include rke2::service
+
+  Class['rke2::install'] -> Class['rke2::config'] -> Class['rke2::service']
+}

modules/rke2/manifests/install.pp

@@ -0,0 +1,10 @@
+# install rke2
+class rke2::install (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+){
+
+  package {"rke2-${node_type}":
+    ensure => installed,
+  }
+
+}

modules/rke2/manifests/params.pp

@@ -0,0 +1,9 @@
+# rke2 params
+class rke2::params (
+  Enum['server', 'agent'] $node_type = 'agent',
+  Stdlib::Absolutepath    $config_file = '/etc/rancher/rke2/config.yaml',
+  Hash                    $config_hash = {},
+  Stdlib::HTTPSUrl        $join_url = 'https://127.0.0.1:9345',
+  Stdlib::Fqdn            $bootstrap_node = 'localhost.localdomain',
+  String                  $node_token = '',
+) {}

modules/rke2/manifests/service.pp

@@ -0,0 +1,13 @@
+# manage rke2 service
+class rke2::service (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+){
+
+  service {"rke2-${node_type}":
+    ensure    => true,
+    enable    => true,
+    subscribe => File[$config_file],
+  }
+
+}

site/profiles/manifests/ceph/conf.pp

@@ -0,0 +1,36 @@
+class profiles::ceph::conf (
+  Hash $config = {}
+) {
+
+  package {[
+    'ceph',
+    'ceph-common'
+  ]:
+    ensure => installed,
+  }
+
+  file {'/etc/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file {'/var/log/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file { '/etc/ceph/ceph.conf':
+    ensure  => file,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0644',
+    content => template('profiles/ceph/conf.erb'),
+    require => Package['ceph-common'],
+  }
+}

site/profiles/manifests/ceph/dashboard.pp

@@ -0,0 +1,20 @@
+class profiles::ceph::dashboard (
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
+){
+
+  # export haproxy balancemember
+  profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_9443":
+    service   => 'be_ceph_dashboard',
+    ipaddress => $ipaddress,
+    ports     => [9443],
+    options   => [
+      "cookie ${facts['networking']['hostname']}",
+      'ssl',
+      'verify none',
+      'check',
+      'inter 2s',
+      'rise 3',
+      'fall 2',
+    ]
+  }
+}

site/profiles/manifests/ceph/rgw.pp

@@ -0,0 +1,41 @@
+class profiles::ceph::rgw (
+  Boolean $enable = true,
+  Hash[String, String] $ceph_client_keys = {},
+  Stdlib::Absolutepath $base_path = '/var/lib/ceph'
+){
+
+  $key = $ceph_client_keys[$facts['networking']['hostname']]
+
+  if $enable {
+
+    include profiles::ceph::conf
+
+    package {'ceph-radosgw':
+      ensure => installed,
+    }
+
+    file { [
+      "${base_path}/radosgw",
+      "${base_path}/radosgw/ceph-${facts['networking']['hostname']}"
+    ]:
+      ensure => directory,
+      owner  => 'ceph',
+      group  => 'ceph',
+      mode   => '0750',
+    }
+
+    file { "${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring":
+      ensure  => file,
+      owner   => 'ceph',
+      group   => 'ceph',
+      mode    => '0750',
+      content => Sensitive("[client.${facts['networking']['hostname']}]\n	key = ${key}\n")
+    }
+
+    service {"ceph-radosgw@${facts['networking']['hostname']}":
+      ensure    => true,
+      enable    => true,
+      subscribe => File["${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring"]
+    }
+  }
+}

site/profiles/manifests/consul/client.pp

@@ -11,6 +11,7 @@ class profiles::consul::client (
   Stdlib::Absolutepath $data_dir           = '/opt/consul',
   Array[Hash]          $node_rules         = [],
   Hash                 $ports              = {},
+  Stdlib::IP::Address  $host_addr          = $facts['networking']['ip'],
 ) {
 
   if $facts['enc_role'] != $members_role {
@@ -42,8 +43,8 @@ class profiles::consul::client (
         'log_level'            => 'INFO',
         'node_name'            => $facts['networking']['fqdn'],
         'retry_join'           => $servers_array,
-        'bind_addr'            => $::facts['networking']['ip'],
+        'bind_addr'            => $host_addr,
-        'advertise_addr'       => $::facts['networking']['ip'],
+        'advertise_addr'       => $host_addr,
         'enable_script_checks' => true,
         'ports'                => $ports,
         'acl'                  => {

site/profiles/manifests/dns/base.pp

@@ -47,7 +47,7 @@ class profiles::dns::base (
   $facts['networking']['interfaces'].each | $interface, $data | {
 
     # exclude those without ipv4 address, lo, docker0 and anycast addresses
-    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ {
+    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ and $interface !~ /^cilium_/  {
 
       # use defaults for the primary_interface
       if $interface == $primary_interface {

site/profiles/manifests/haproxy/balancemember.pp

@@ -3,6 +3,7 @@ define profiles::haproxy::balancemember (
   String              $service,
   Array[Stdlib::Port] $ports,
   Array               $options = ['check'],
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
 ) {
 
   $location_environment = "${facts['country']}-${facts['region']}-${facts['environment']}"
@@ -12,7 +13,7 @@ define profiles::haproxy::balancemember (
     listening_service => $service,
     ports             => $ports,
     server_names      => $facts['networking']['hostname'],
-    ipaddresses       => $facts['networking']['ip'],
+    ipaddresses       => $ipaddress,
     options           => $options,
     tag               => $balancemember_tag,
   }

site/profiles/manifests/metrics/grafana.pp

@@ -11,6 +11,7 @@ class profiles::metrics::grafana (
   String    $db_pass     = fqdn_rand_string(16),
   Stdlib::Host $db_host  = '127.0.0.1',
   Stdlib::Port $db_port  = 5432,
+  Hash      $plugins     = {}
 ) {
 
   # set the fqdn
@@ -108,6 +109,7 @@ class profiles::metrics::grafana (
   class { 'grafana':
     cfg      => $cfg,
     ldap_cfg => $ldap_cfg,
+    plugins  => $plugins,
   }
 
   # fix the package provided systemd service

site/profiles/templates/ceph/conf.erb

@@ -0,0 +1,9 @@
+# Managed by Puppet in profiles::ceph::conf
+<% @config.each do |section, settings| -%>
+[<%= section %>]
+<% settings.each do |key, value| -%>
+<%# Convert booleans and numbers to strings, leave strings untouched %>
+<%= key %> = <%= value.is_a?(TrueClass) ? 'true' : value.is_a?(FalseClass) ? 'false' : value %>
+<% end -%>
+
+<% end -%>

site/roles/manifests/infra/ceph/rgw.pp

@@ -0,0 +1,10 @@
+# a role to deploy the ceph rgw
+class roles::infra::ceph::rgw {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}