fix: cattle-system namespace

- cattle-system namespace is created earlier than helm - leave namespaces.yaml to manage cattle-system namespace (required before installing helm/rancher)
feat: define node-token from puppet (#398 )
2025-09-20 22:35:35 +10:00 · 2025-09-20 22:25:56 +10:00 · 2025-09-20 18:40:18 +10:00 · 2025-09-20 17:40:41 +10:00 · 2025-09-14 20:59:39 +10:00 · 2025-09-14 13:27:49 +10:00
51 changed files with 1775 additions and 76 deletions
@@ -29,3 +29,21 @@ these steps are required when adding additional puppet masters, as the subject a

    sudo systemctl start puppetserver
    sudo cp /root/current_crl.pem /etc/puppetlabs/puppet/ssl/crl.pem
+
+
+## troubleshooting
+
+### Issue 1:
+
+    [sysadmin@ausyd1nxvm2056 ~]$ sudo puppet agent -t
+    Error: The CRL issued by 'CN=Puppet CA: prodinf01n01.main.unkin.net' is missing
+
+Find another puppetserver that IS working, copy the `/etc/puppetlabs/puppet/ssl/crl.pem` to this host, run puppet again.
+
+
+### Issue 2:
+
+    [sysadmin@ausyd1nxvm2097 ~]$ sudo puppet agent -t
+    Error: Failed to parse CA certificates as PEM
+
+The puppet-agents CA cert `/etc/puppetlabs/puppet/ssl/certs/ca.pem` is empty or missing. Grab it from any other host. Run puppet again.
@@ -155,6 +155,9 @@ lookup_options:
  zfs::datasets:
    merge:
      strategy: deep
+  rke2::config_hash:
+    merge:
+      strategy: deep

 facts_path: '/opt/puppetlabs/facter/facts.d'

@@ -368,6 +371,28 @@ profiles::ceph::client::mons:
  - 10.18.15.1
  - 10.18.15.2
  - 10.18.15.3
+
+profiles::ceph::conf::config:
+  global:
+    auth_client_required: 'cephx'
+    auth_cluster_required: 'cephx'
+    auth_service_required: 'cephx'
+    fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+    mon_allow_pool_delete: true
+    mon_initial_members: 'prodnxsr0009,prodnxsr0010,prodnxsr0011,prodnxsr0012,prodnxsr0013'
+    mon_host: '198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13'
+    ms_bind_ipv4: true
+    ms_bind_ipv6: false
+    osd_crush_chooseleaf_type: 1
+    osd_pool_default_min_size: 2
+    osd_pool_default_size: 3
+    osd_pool_default_pg_num: 128
+    public_network: >
+      198.18.23.1/32,198.18.23.2/32,198.18.23.3/32,198.18.23.4/32,
+      198.18.23.5/32,198.18.23.6/32,198.18.23.7/32,198.18.23.8/32,
+      198.18.23.9/32,198.18.23.10/32,198.18.23.11/32,198.18.23.12/32,
+      198.18.23.13/32
+
 #profiles::base::hosts::additional_hosts:
 #  - ip: 198.18.17.9
 #    hostname: prodinf01n09.main.unkin.net
@@ -10,6 +10,7 @@ profiles::haproxy::dns::vrrp_cnames:
  - git.unkin.net
  - fafflix.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net

 profiles::haproxy::mappings:
  fe_http:
@@ -27,6 +28,7 @@ profiles::haproxy::mappings:
      - 'fafflix.unkin.net be_jellyfin'
      - 'git.unkin.net be_gitea'
      - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'
  fe_https:
    ensure: present
    mappings:
@@ -42,6 +44,7 @@ profiles::haproxy::mappings:
      - 'fafflix.unkin.net be_jellyfin'
      - 'git.unkin.net be_gitea'
      - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'

 profiles::haproxy::frontends:
  fe_http:
@@ -62,6 +65,7 @@ profiles::haproxy::frontends:
        - 'acl_fafflix req.hdr(host) -i fafflix.unkin.net'
        - 'acl_gitea req.hdr(host) -i git.unkin.net'
        - 'acl_grafana req.hdr(host) -i grafana.unkin.net'
+        - 'acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net'
        - 'acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24'
      use_backend:
        - "%[req.hdr(host),lower,map(/etc/haproxy/fe_https.map,be_default)]"
@@ -79,6 +83,7 @@ profiles::haproxy::frontends:
        - 'set-header X-Frame-Options DENY if acl_fafflix'
        - 'set-header X-Frame-Options DENY if acl_gitea'
        - 'set-header X-Frame-Options DENY if acl_grafana'
+        - 'set-header X-Frame-Options DENY if acl_ceph_dashboard'
        - 'set-header X-Content-Type-Options nosniff'
        - 'set-header X-XSS-Protection 1;mode=block'

@@ -262,6 +267,26 @@ profiles::haproxy::backends:
      redirect: 'scheme https if !{ ssl_fc }'
      stick-table: 'type ip size 200k expire 30m'
      stick: 'on src'
+  be_ceph_dashboard:
+    description: Backend for Ceph Dashboard from Mgr instances
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-check:
+        - expect status 200
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 9443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+      stick-table: 'type ip size 200k expire 30m'
+      stick: 'on src'

 profiles::haproxy::certlist::enabled: true
 profiles::haproxy::certlist::certificates:
@@ -276,6 +301,7 @@ profiles::haproxy::certlist::certificates:
  - /etc/pki/tls/letsencrypt/fafflix.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/letsencrypt/git.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/letsencrypt/grafana.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/dashboard.ceph.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/vault/certificate.pem

 # additional altnames
@@ -303,3 +329,4 @@ certbot::client::domains:
  - fafflix.unkin.net
  - git.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net
@@ -70,7 +70,7 @@ profiles::nginx::simpleproxy::locations:
  arrstack_web_external:
    location_satisfy: any
    location_allow:
-      - 198.18.13.47
-      - 198.18.13.50
-      - 198.18.13.51
-      - 198.18.13.52
+      - 198.18.26.161
+      - 198.18.27.131
+      - 198.18.28.165
+      - 198.18.29.32
@@ -211,6 +211,18 @@ glauth::users:
    loginshell: '/bin/bash'
    homedir: '/home/waewak'
    passsha256: 'd9bb99634215fe031c3bdca94149a165192fe8384ecaa238a19354c2f760a811'
+  debvin:
+    user_name: 'debvin'
+    givenname: 'Debbie'
+    sn: 'Vincent'
+    mail: 'debvin@users.main.unkin.net'
+    uidnumber: 20009
+    primarygroup: 20000
+    othergroups:
+      - 20010 # jelly
+    loginshell: '/bin/bash'
+    homedir: '/home/debvin'
+    passsha256: 'cdac05ddb02e665d4ea65a974995f38a10236bc158731d92d78f6cde89b294a1'

 glauth::services:
  svc_jellyfin:
@@ -0,0 +1,8 @@
+---
+
+profiles::ceph::rgw::ceph_client_keys:
+  ausyd1nxvm2115: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAovOUUtTm/9nXWj6D+kVhmbZedVVkA5N80ULNYqISrv0A+KlXCx/V2sB56SpJ9eL6aPHdC+CKkrtrrdo1h2j2em7wTA5ghhbFVF0NIS7bbn0jzpl5YqbUrNxOtDSpjzX1aCGnyMvw69Lp+NcHwxIj+1XFgK6335138s9wbX3HmYF3jcsQkMqTzynELe1OQPWFXVKTjUfFFLdCQOFryp8UY8L9j/PpV6wd4w6p7R6eXhX21rjSaN4aqN1zjsnF2OVhL8Ge0QxMhePWKGOqsUfi72kh3II028DjqU0DcZQvoxnoqRPyUUjysH0nTKoLeXOGNgJdphY1dHBJ+SJnw2gqQDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB9wA3ZJjmU95W+9r8va5uagDAMAv2APfdzrPnAU7NJPL+IW08osGuQWWamqF+XGVeHRoBmoFKwZ7grYRV2e3aabyc=]
+  ausyd1nxvm2116: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAf5ksy/pyUZSwTh+HiKw+1Uhj16A0DVZEKAbkKUQzXVmc+QpL4Dn7YBoXlEwrY8CcsTrxTQjADvtu9FC3o34QIdh06noSgYYA+7fna2A+9+oYoNtgwC3b8LeglxO/SQ9dKoJ90jRtmlw5P/CtrxA2RelMK6FNRekp1CaWMM4q20fJGgr/E33vgx38UJyp4/q0bTu2lLehCuDUP80j3XGbSNZ2snfYdIo91Cl+nSxLSU2TdnFpWaabsH19HwDnkWGiILlLBVvvhY7copCxs5DS1ueoOTCsqnWSrTrBMJjnu7WZd/s4NLw/0q/UP5xcFA51caY3Kv+sI6bfIYkNoLazwDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDrCF16nrtukRPamx1VbGY+gDAK5dw0kV8MpATpwxTkJG6JtlFlwdpU9THs1bNwqSRD1ZhEWxQeWwsyyTtjUXi4bP8=]
+  ausyd1nxvm2117: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAKtvsgDk2QTmL8flBBE4nA43sRSnroq4I6T2CUAYv/lRdzCCWvE961o/51zyQEz8L5QyDo7L3gcGYIBqthYjRe9Gp5a5d4qds0qskupgQKnb0KR2RLFVUtH5vxHqyJZHjXaP+PQreyRoSIfRWXAdoZu544FeJ9DKKbmEzZaH5B2OdDMrf4Ufuud0maEAw0PJthS//ghCfGi74F1xlJnIWVvMhp66b0iMxC+ACClEHunG3oKx7M/w05HllG0wcxPTg4PFrbnFXjRuIxsykF9aVHJkRnCdgbMXRM4o6FrYyZRR74F1HKRujFCUA7kYWDKLxHxJpYCvCHp4HMhfzjs824zBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBbaSig6kgVVNfmSI53sNimgDDQ5O10Dzfa7S7RdJVLLUFBaZ5TG2g3Bwmy0k3wKZvABYMuYyOxQdfk6eMsKC+sC5w=]
+  ausyd1nxvm2118: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAdKl0ude3ZxW0ihyA758mpQp43qZvzTI2Bp83WzCp2ifJCgAIjRWdd01P64rSHaa4lus/wqB9jxg38g6DrN4ejX56Y/CJ6GQKxz6b1BO5nDfsLx6QEzCt+cfg5d/PPoTtEpz2VSvDfxFUrHiABA6++Sqzb9Og+nQCFMYJD3NHCk67QpkjPGQ/ejZk4MNXZQVCfKOlFqay/fF0jEmQixFOlX/Fdm9UoKttbrKluUmzpaVUzfGRaTTFVgzc3x2t/z1q1k0P7ClI9Uu02kUXpFVs9LPX99Zc2GtrnP06mYqqARhWF1NMK0zlmxtKpfObahRP/HmtI3fgnQsU1Cpwah0emTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAILqpYx3FKY3xXLJRu2oDlgDCIOXeX6hxpu0qpj5c/9jMUSeV2DIydnxO+MiT3mceS50ip8B+zGQy5UedPmLt36Zs=]
+  ausyd1nxvm2119: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEASlZglUxazp+9azfV3QkgRv+ACo+MH0RO5b18blbelgdmr38iwK7MwwEFpEfVJEyc/ph9RunWwrMmofDQHj5bBribfzZ2pH2CGiOrR0i5lZMtN0yQXPBA/+jm1Pi1AWGJLtoquuhMbibuHOTiXwBCBVrHHHaFTR5Xt34ABN/p/mCaG+N9nWux93msHCCextCalKBMmPhmI2q6HodfjanEVgYAe3/5hRPnpsi6IGSDNGygsTC3MG+hjGMpNF8izbwk9Lpzn6kY51aeNxI2ed9Jm8UZ/k+8b+o7ZQyWIBbf7DTFpEzk4G46puaDbXIorBWQ4azCjN3gt8VB91hwihtzcDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB6U+9z4cSzMTA1z9bmoX82gDBfy5zbRPK8GxImJo6evecMOTtaY2c4aEnESXtGBCS02enmxljv9dv1UYQD0/a6S3A=]
@@ -0,0 +1,60 @@
+---
+hiera_include:
+  - profiles::ceph::rgw
+  - profiles::nginx::simpleproxy
+
+profiles::ceph::rgw::enable: true
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - radosgw.main.unkin.net
+  - radosgw.service.consul
+  - radosgw.query.consul
+  - "radosgw.service.%{facts.country}-%{facts.region}.consul"
+
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+
+# manage a simple nginx reverse proxy
+profiles::nginx::simpleproxy::nginx_vhost: 'radosgw.service.consul'
+profiles::nginx::simpleproxy::nginx_aliases:
+  - radosgw.service.au-syd1.consul
+profiles::nginx::simpleproxy::proxy_port: 7480
+profiles::nginx::simpleproxy::proxy_path: '/'
+nginx::client_max_body_size: 100M
+
+# manage consul service
+consul::services:
+  radosgw:
+    service_name: 'radosgw'
+    address: "%{facts.networking.ip}"
+    port: 443
+    checks:
+      - id: 'radosgw_https_check'
+        name: 'RADOSGW HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: radosgw
+    disposition: write
@@ -6,6 +6,7 @@ hiera_include:
  - zfs
  - profiles::ceph::node
  - profiles::ceph::client
+  - profiles::ceph::dashboard
  - profiles::storage::cephfsvols
  - exporters::frr_exporter

@@ -36,6 +37,7 @@ profiles::ssh::sign::principals:
  - "%{facts.networking.interfaces.enp3s0.ip}"

 # configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
 consul::services:
  incus:
    service_name: 'incus'
@@ -121,6 +123,9 @@ profiles::yum::global::repos:
 # dns
 profiles::dns::base::primary_interface: loopback0

+# dashboard/haproxy
+profiles::ceph::dashboard::ipaddress: "%{hiera('networking_loopback0_ip')}"
+
 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true
@@ -150,12 +155,11 @@ networking::interfaces:
    netmask: 255.255.255.255
    mtu: 1500

-exporters::node_exporter::addr: "%{hiera('networking_loopback0_ip')}"
-
 # frrouting
 exporters::frr_exporter::enable: true
-exporters::frr_exporter::addr: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_redistribute:
  - connected
 frrouting::ospfd_interfaces:
@@ -0,0 +1 @@
+rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAVPo7WjzlJQQhxefhV1bu1/+Jo/LY3gMmVGicDnDloGQd1jbdDtNz9wi6Hqqht1xQPdn22XmvvrVXtPhsdjDCGqWxmfZ0qWQVl1Ju2WIh5WsyyZgdE96k2+y7Cg5Dl0brX2m9YSZfow5BF8J8EnCDdRZncOCtFl/SU8ipPEq2uJJR+Y9sJv6aJflnFLCEYgJNbZY9ljMcs5ssJ21VpIqWYA0Z6vKVqOUeIWWKbYZUIoEml7sj3ktmw3loMYA6ED0/nzyYvRVizTvtGXl3IWGVDSt8rQ/kNhzKqURVOsgwfbt7un4n2Kxkreiydj3R6PbLdkpHdtu25dbjue8HLQkc4zCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQMmIatr2oBei35+evghIkLIBwWT6F1WPDV3PA/QYNusfYcI1KrMRYFlDYyEkg/Tf+0gOpty9rdnYL+MQO2fTU9Br+INTr1oJcxJp/Lqap2+NibmBfZcLUIMn3q1S/jZp9BGQTk6RSwODqQ2x5GxDATinrtiUR4TXIIaiaP3KWlP8A7g==]
@@ -0,0 +1,168 @@
+---
+hiera_include:
+  - profiles::selinux::setenforce
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - exporters::frr_exporter
+  - frrouting
+  - rke2
+
+# manage rke2
+rke2::bootstrap_node: prodnxsr0001.main.unkin.net
+rke2::join_url: https://join-k8s.service.consul:9345
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  write-kubeconfig-mode: 644
+  kubelet-arg:
+    - '--node-status-update-frequency=4s'
+    - '--max-pods=100'
+  node-label:
+    - "region=%{facts.region}"
+    - "country=%{facts.country}"
+    - "asset=%{facts.dmi.product.serial_number}"
+    - "zone=%{zone}"
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
+
+profiles::packages::include:
+  bridge-utils: {}
+  cephadm: {}
+
+profiles::selinux::setenforce::mode: disabled
+
+profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_package: false
+profiles::ceph::client::manage_ceph_paths: false
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+profiles::ceph::client::mons:
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://packagerepo.service.consul/rke2/rhel%{facts.os.release.major}/common-daily/x86_64/os/
+    gpgkey: https://packagerepo.service.consul/rke2/rhel%{facts.os.release.major}/common-daily/x86_64/os/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://packagerepo.service.consul/rke2/rhel%{facts.os.release.major}/1.33-daily/x86_64/os/
+    gpgkey: https://packagerepo.service.consul/rke2/rhel%{facts.os.release.major}/1.33-daily/x86_64/os/public.key
+    mirrorlist: absent
+
+# dns
+profiles::dns::base::primary_interface: loopback0
+
+# networking
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  "%{hiera('networking_1000_iface')}":
+    type: physical
+    ipaddress: "%{hiera('networking_1000_ip')}"
+    gateway: 198.18.15.254
+    txqueuelen: 10000
+    forwarding: true
+  "%{hiera('networking_2500_iface')}":
+    type: physical
+    ipaddress: "%{hiera('networking_2500_ip')}"
+    mtu: 1500
+    txqueuelen: 10000
+    forwarding: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback1:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback1_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback2:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback2_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: frr_exporter
+    disposition: write
+
+# frrouting
+exporters::frr_exporter::enable: true
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  "%{hiera('networking_1000_iface')}":
+    area: 0.0.0.0
+  "%{hiera('networking_2500_iface')}":
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+  loopback1:
+    area: 0.0.0.0
+  loopback2:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+frrouting::ospf_exclude_k8s_enable: true
+frrouting::k8s_cluster_cidr: '10.42.0.0/16'  # RKE2 cluster-cidr (pods)
+frrouting::k8s_service_cidr: '10.43.0.0/16'  # RKE2 service-cidr
+
+# add loopback interfaces to ssh list
+ssh::server::options:
+  ListenAddress:
+    - "%{hiera('networking_loopback0_ip')}"
+    - "%{hiera('networking_1000_ip')}"
+    - "%{hiera('networking_2500_ip')}"
+
+profiles::ssh::sign::principals:
+  - "%{hiera('networking_loopback0_ip')}"
+  - "%{hiera('networking_1000_ip')}"
+  - "%{hiera('networking_2500_ip')}"
@@ -1,10 +1,3 @@
 ---
-# networking
-systemd::manage_networkd: true
-systemd::manage_all_network_files: true
-networking::interfaces:
-  eth0:
-    type: physical
-    forwarding: true
-    dhcp: true
-    mtu: 1500
+# manage rke2
+rke2::node_type: agent
@@ -0,0 +1,2 @@
+---
+rke2::csi_ceph_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEApQ371O4nGSrFB5tOZFTSJP+kJj3wJyEcWiNfonYA5LmbaMnQ6pUortec1519WHMICpSWdpq3O8frivm2CK3taYoKczeTzbsFTxvVp7s6gIZJUsCeqGHuq81YyjPtJE+Yy5IOBJjhe/8ECkEFNr0JlhwKBPWfTx5hHOzRdkGlN464weGFQtCI8UgdGe7AWEePG+u3e4RL+xCriw5tfuqMeeo+isDwVf30nK9NxsnmliOd/+jNW+GrtzycHAeokQOKnxfgrKll5Y5+npy5WueuSCEw1E+Io0NI/4Jthi7zu24UQu0KT8iRsqhuD5mr1ymvCNREnvCcVWt8VVRTGXQV+TBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDS8VXZM3wEAFRALB/Fa19dgDCTRYhU8YY4g9zREP7epY9x2MRWaTT84Jx9w5Dc/XWaRrmL4yL2sK+QHSy2057jHzo=]
@@ -1,42 +1,99 @@
 ---
-profiles::pki::vault::alt_names:
-  - k8s-control.service.consul
-  - k8s-control.query.consul
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
-
-profiles::ssh::sign::principals:
-  - k8s-control.service.consul
-  - k8s-control.query.consul
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
+# manage rke2
+rke2::node_type: server
+rke2::helm_install: true
+rke2::helm_repos:
+  rancher-stable: https://releases.rancher.com/server-charts/stable
+  purelb: https://gitlab.com/api/v4/projects/20400619/packages/helm/stable
+  jetstack: https://charts.jetstack.io
+  harbor: https://helm.goharbor.io
+  traefik: https://traefik.github.io/charts
+  hashicorp: https://helm.releases.hashicorp.com
+rke2::csi_ceph_enable: true
+rke2::csi_ceph_clusterid: de96a98f-3d23-465a-a899-86d3d67edab8
+rke2::csi_ceph_poolname: kubernetes
+rke2::csi_ceph_monitors:
+  - 198.18.23.9:6789
+  - 198.18.23.10:6789
+  - 198.18.23.11:6789
+  - 198.18.23.12:6789
+  - 198.18.23.13:6789
+rke2::csi_ceph_files:
+  - ceph-csi-nodeplugin-rbac
+  - ceph-csi-provisioner-rbac
+  - ceph-csi-rbdplugin-provisioner
+  - ceph-csi-rbdplugin
+rke2::csi_ceph_templates:
+  - ceph-csi-config
+  - ceph-csi-secret
+rke2::extra_config_files:
+  - namespaces
+  - rke2-canal-config
+  - purelb-config
+  - ingres-lb-nginx
+  - ingres-route-rancher
+rke2::config_hash:
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "join-k8s.service.consul"
+    - "api-k8s.service.consul"
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: canal
+  cluster-cidr: 10.42.0.0/16
+  service-cidr: 10.43.0.0/16
+  cluster-dns: 10.43.0.10
+  etcd-arg: "--quota-backend-bytes 2048000000"
+  etcd-snapshot-schedule-cron: "0 3 * * *"
+  etcd-snapshot-retention: 10
+  kube-apiserver-arg:
+    - '--default-not-ready-toleration-seconds=30'
+    - '--default-unreachable-toleration-seconds=30'
+  kube-controller-manager-arg:
+    - '--node-monitor-period=4s'
+  protect-kernel-defaults: true
+  disable-kube-proxy: false

 # configure consul service
 consul::services:
-  k8s-control:
-    service_name: 'k8s-control'
-    tags:
-      - 'k8s'
-      - 'container'
+  api-k8s:
+    service_name: 'api-k8s'
    address: "%{facts.networking.fqdn}"
    port: 6443
    checks:
-      - id: 'k8s-control_https_check'
-        name: 'k8s-control HTTPS Check'
-        http: "https://%{facts.networking.fqdn}:6443"
-        method: 'GET'
-        tls_skip_verify: true
+      - id: 'api-k8s_livez_check'
+        name: 'api-k8s livez Check'
+        args:
+          - sudo
+          - /usr/local/bin/check_k8s_api.sh
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'rke2_tcp_check_9345'
+        name: 'rke2 TCP Check 9345'
+        tcp: "%{hiera('networking_loopback0_ip')}:9345"
        interval: '10s'
        timeout: '1s'
 profiles::consul::client::node_rules:
  - resource: service
-    segment: k8s-control
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
    disposition: write

-# networking
-systemd::manage_networkd: true
-systemd::manage_all_network_files: true
-networking::interfaces:
-  eth0:
-    type: physical
-    forwarding: true
-    dhcp: true
-    mtu: 1500
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"
+
+sudo::configs:
+  consul-checks:
+    priority: 20
+    content: |
+      consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh
@@ -5,6 +5,24 @@ hiera_include:
  - profiles::ceph::node
  - profiles::ceph::client
  - exporters::frr_exporter
+  - profiles::rke2::node
+
+# manage rke2
+profiles::rke2::node::servers:
+  - prodnxsr0001.main.unkin.net
+  - prodnxsr0002.main.unkin.net
+  - prodnxsr0003.main.unkin.net
+
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: cilium

 # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
 python::manage_dev_package: false
@@ -25,6 +43,7 @@ profiles::ceph::client::mons:
  - 198.18.23.11
  - 198.18.23.12
  - 198.18.23.13
+
 # additional repos
 profiles::yum::global::repos:
  ceph:
@@ -55,6 +74,20 @@ profiles::yum::global::repos:
    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
    mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent

 # dns
 profiles::dns::base::primary_interface: loopback0
@@ -91,18 +124,47 @@ networking::interfaces:
    netmask: 255.255.255.255
    mtu: 1500

-# consul
+# configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+consul::services:
+  api-k8s:
+    service_name: 'api-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 6443
+    checks:
+      - id: 'api-k8s_https_check'
+        name: 'api-k8s HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:6443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'etcd_tcp_check_9345'
+        name: 'ETCD TCP Check 9345'
+        tcp: "%{facts.networking.fqdn}:9345"
+        interval: '10s'
+        timeout: '1s'
 profiles::consul::client::node_rules:
+  - resource: service
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
+    disposition: write
  - resource: service
    segment: frr_exporter
    disposition: write

-exporters::node_exporter::addr: "%{hiera('networking_loopback0_ip')}"
-
 # frrouting
 exporters::frr_exporter::enable: true
-exporters::frr_exporter::addr: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_redistribute:
  - connected
 frrouting::ospfd_interfaces:
@@ -130,3 +192,8 @@ profiles::ssh::sign::principals:
  - "%{hiera('networking_loopback0_ip')}"
  - "%{hiera('networking_1000_ip')}"
  - "%{hiera('networking_2500_ip')}"
+
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"
@@ -11,6 +11,9 @@ profiles::metrics::grafana::db_name: "%{hiera('profiles::sql::postgresdb::dbname
 profiles::metrics::grafana::db_user: "%{hiera('profiles::sql::postgresdb::dbuser')}"
 profiles::metrics::grafana::db_pass: "%{hiera('profiles::sql::postgresdb::dbpass')}"
 profiles::metrics::grafana::pgsql_backend: true
+profiles::metrics::grafana::plugins:
+  victoriametrics-logs-datasource:
+    ensure: present

 # additional altnames
 profiles::pki::vault::alt_names:
@@ -15,3 +15,4 @@ certbot::domains:
  - fafflix.unkin.net
  - git.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net
@@ -283,6 +283,20 @@ profiles::reposync::repos_list:
    release: 'rhel9'
    baseurl: 'https://download.postgresql.org/pub/repos/yum/17/redhat/rhel-9-x86_64/'
    gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  rke2_common_el9:
+    repository: 'common'
+    description: 'RKE2 common RHEL 9'
+    osname: 'rke2'
+    release: "rhel9"
+    baseurl: "https://rpm.rancher.io/rke2/latest/common/centos/9/noarch"
+    gpgkey: "https://rpm.rancher.io/public.key"
+  rke2_1_33_el9:
+    repository: '1.33'
+    description: 'RKE2 1.33 RHEL 9'
+    osname: 'rke2'
+    release: "rhel9"
+    baseurl: "https://rpm.rancher.io/rke2/latest/1.33/centos/9/x86_64"
+    gpgkey: "https://rpm.rancher.io/public.key"
  zfs_dkms_rhel8:
    repository: 'dkms'
    description: 'ZFS DKMS RHEL 8'
@@ -4,7 +4,6 @@ class exporters::frr_exporter (
  String  $group          = 'frr_exporter',
  Boolean $manage_user    = true,
  Boolean $manage_service = true,
-  Stdlib::Host $addr      = $facts['networking']['ip'],
  Stdlib::Port $port      = 9342,
  Stdlib::Absolutepath $exec_path  = '/usr/bin/frr_exporter',
  Stdlib::Absolutepath $socket_dir = '/var/run/frr',
@@ -17,6 +16,14 @@ class exporters::frr_exporter (
      ensure => installed,
    }

+    # ensure the frr_exporter user can read the directory
+    file { $socket_dir:
+      ensure => directory,
+      owner  => 'frr',
+      group  => 'frr',
+      mode   => '0751',
+    }
+
    # manage the user/group
    if $manage_user {
      group { $group:
@@ -46,7 +53,7 @@ class exporters::frr_exporter (
    # manage consul service
    consul::service { 'frr_exporter':
      service_name => 'frr_exporter',
-      address      => $addr,
+      address      => $facts['networking']['ip'],
      port         => $port,
      tags         => [
        'metrics',
@@ -4,7 +4,6 @@ class exporters::node_exporter (
  String  $group          = 'node_exporter',
  Boolean $manage_user    = true,
  Boolean $manage_service = true,
-  Stdlib::Host $addr      = $facts['networking']['ip'],
  Stdlib::Port $port      = 9100,
  Stdlib::Absolutepath $exec_path     = '/usr/bin/node_exporter',
  Boolean $cleanup_old_node_exporter  = false,
@@ -59,7 +58,7 @@ class exporters::node_exporter (
    # manage consul service
    consul::service { 'node_exporter':
      service_name => 'node_exporter',
-      address      => $addr,
+      address      => $facts['networking']['ip'],
      port         => $port,
      tags         => [
        'metrics',
@@ -14,8 +14,18 @@ class frrouting (
  Optional[String] $mpls_ldp_router_id       = undef,
  Optional[String] $mpls_ldp_transport_addr  = undef,
  Array[String] $mpls_ldp_interfaces         = [],
+  Boolean $ospf_preferred_source_enable      = false,
+  Optional[Stdlib::IP::Address] $ospf_preferred_source = undef,
+  Boolean $ospf_exclude_k8s_enable           = false,
+  Optional[Stdlib::IP::Address::V4::CIDR] $k8s_cluster_cidr = undef, # pod/cluster CIDR (e.g. 10.42.0.0/16)
+  Optional[Stdlib::IP::Address::V4::CIDR] $k8s_service_cidr = undef, # service CIDR (e.g. 10.43.0.0/16)
 ) {

+    # sanity check
+  if $ospf_exclude_k8s_enable and $k8s_cluster_cidr == undef and $k8s_service_cidr == undef {
+    warning('frrouting: ospf_exclude_k8s_enable is true but no k8s_*_cidr provided; nothing will be filtered.')
+  }
+
  $daemons_defaults = {
    'bgpd'    => false,
    'ospfd'   => true,
@@ -2,6 +2,7 @@
 frr defaults traditional
 hostname <%= @hostname %>
 no ipv6 forwarding
+
 <% @ospfd_interfaces.each do |iface, params| -%>
 interface <%= iface %>
 <% if params['area'] -%>
@@ -12,12 +13,31 @@ interface <%= iface %>
 <% end -%>
 exit
 <% end -%>
+
+<%# ---- K8s exclude (prefix-list + route-map) ---- -%>
+<% if @ospf_exclude_k8s_enable && (@k8s_cluster_cidr || @k8s_service_cidr) -%>
+! Do not redistribute Kubernetes Pod/Service CIDRs
+ip prefix-list K8S-BLOCK seq 5 permit <%= @k8s_cluster_cidr %> le 32<% if !@k8s_cluster_cidr %>  ! (unset)<% end %>
+<% if @k8s_service_cidr -%>
+ip prefix-list K8S-BLOCK seq 10 permit <%= @k8s_service_cidr %> le 32
+<% end -%>
+route-map CONNECTED-NON-K8S deny 5
+ match ip address prefix-list K8S-BLOCK
+exit
+route-map CONNECTED-NON-K8S permit 100
+exit
+<% end -%>
+
 router ospf
 ospf router-id <%= @ospfd_router_id %>
 log-adjacency-changes detail
 <% @ospfd_redistribute.each do |type| -%>
+<% if @ospf_exclude_k8s_enable && type == 'connected' && (@k8s_cluster_cidr || @k8s_service_cidr) -%>
+ redistribute connected route-map CONNECTED-NON-K8S
+<% else -%>
 redistribute <%= type %>
 <% end -%>
+<% end -%>
 <% @ospfd_networks.each do |network| -%>
 network <%= network %>
 <% end -%>
@@ -31,6 +51,8 @@ router ospf
 mpls-te inter-as area 0.0.0.0
 <% end -%>
 exit
+
+<%# ---- MPLS/LDP config ---- -%>
 <% if @mpls_ldp_router_id and @mpls_ldp_transport_addr and @mpls_ldp_interfaces.any? -%>
 mpls ldp
 router-id <%= @mpls_ldp_router_id %>
@@ -43,3 +65,13 @@ mpls ldp
 exit-address-family
 exit
 <% end -%>
+
+<%# ---- Preferred OSPF source ---- -%>
+<% if @ospf_preferred_source_enable && @ospf_preferred_source -%>
+ip prefix-list ANY seq 5 permit 0.0.0.0/0 le 32
+route-map OSPF-SRC permit 10
+ match ip address prefix-list ANY
+ set src <%= @ospf_preferred_source %>
+exit
+ip protocol ospf route-map OSPF-SRC
+<% end -%>
@@ -0,0 +1,48 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rbd-csi-nodeplugin
+  namespace: ceph-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin
+subjects:
+  - kind: ServiceAccount
+    name: rbd-csi-nodeplugin
+    namespace: ceph-csi
+roleRef:
+  kind: ClusterRole
+  name: rbd-csi-nodeplugin
+  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,125 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rbd-csi-provisioner
+  namespace: ceph-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-external-provisioner-runner
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch", "create"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["get", "list", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts/token"]
+    verbs: ["create"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotcontents/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumegroupreplicationcontents"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumegroupreplicationclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: rbd-csi-provisioner
+    namespace: ceph-csi
+roleRef:
+  kind: ClusterRole
+  name: rbd-external-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-external-provisioner-cfg
+  namespace: ceph-csi
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "update", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-provisioner-role-cfg
+  namespace: ceph-csi
+subjects:
+  - kind: ServiceAccount
+    name: rbd-csi-provisioner
+    namespace: ceph-csi
+roleRef:
+  kind: Role
+  name: rbd-external-provisioner-cfg
+  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,124 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-rbdplugin-provisioner
+  namespace: ceph-csi
+  labels:
+    app: csi‑metrics
+spec:
+  selector:
+    app: csi‑rbdplugin‑provisioner
+  ports:
+    - name: http‑metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8680
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: csi‑rbdplugin‑provisioner
+  namespace: ceph-csi
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: csi‑rbdplugin‑provisioner
+  template:
+    metadata:
+      labels:
+        app: csi‑rbdplugin‑provisioner
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - csi‑rbdplugin‑provisioner
+              topologyKey: "kubernetes.io/hostname"
+      serviceAccountName: rbd‑csi‑provisioner
+      priorityClassName: system‑cluster‑critical
+      containers:
+        - name: csi‑rbdplugin
+          image: quay.io/cephcsi/cephcsi:v3.15
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=rbd"
+            - "--controllerserver=true"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
+            - "--v=5"
+            - "--drivername=rbd.csi.ceph.com"
+            - "--pidlimit=-1"
+            - "--rbdhardmaxclonedepth=8"
+            - "--rbdsoftmaxclonedepth=4"
+            - "--enableprofiling=false"
+            - "--setmetadata=true"
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi-provisioner.sock
+            - name: CSI_ADDONS_ENDPOINT
+              value: unix:///csi/csi-addons.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: host-dev
+              mountPath: /dev
+            - name: host-sys
+              mountPath: /sys
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: ceph-csi-encryption-kms-config
+              mountPath: /etc/ceph-csi-encryption-kms-config/
+            - name: ceph-config
+              mountPath: /etc/ceph/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+        # snapshotter & other sidecars omitted in this snippet for brevity
+      volumes:
+        - name: socket-dir
+          emptyDir:
+            medium: Memory
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config
+        - name: ceph-csi-encryption-kms-config
+          configMap:
+            name: ceph-csi-encryption-kms-config
+        - name: ceph-config
+          configMap:
+            name: ceph-config
+        - name: keys-tmp-dir
+          emptyDir:
+            medium: Memory
+        # and other volumes as in the original
@@ -0,0 +1,155 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: csi‑rbdplugin
+  namespace: ceph-csi
+spec:
+  selector:
+    matchLabels:
+      app: csi‑rbdplugin
+  template:
+    metadata:
+      labels:
+        app: csi‑rbdplugin
+    spec:
+      serviceAccountName: rbd‑csi‑nodeplugin
+      hostNetwork: true
+      hostPID: true
+      priorityClassName: system‑node‑critical
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: csi‑rbdplugin
+          securityContext:
+            privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+          allowPrivilegeEscalation: true
+          image: quay.io/cephcsi/cephcsi:v3.15
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--pluginpath=/var/lib/kubelet/plugins"
+            - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/"
+            - "--type=rbd"
+            - "--nodeserver=true"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
+            - "--v=5"
+            - "--drivername=rbd.csi.ceph.com"
+            - "--enableprofiling=false"
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: CSI_ADDONS_ENDPOINT
+              value: unix:///csi/csi-addons.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: host-dev
+              mountPath: /dev
+            - name: host-sys
+              mountPath: /sys
+            - name: host-mount
+              mountPath: /run/mount
+            - name: etc-selinux
+              mountPath: /etc/selinux
+              readOnly: true
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: plugin-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: "Bidirectional"
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+            - name: ceph-logdir
+              mountPath: /var/log/ceph
+            - name: ceph-config
+              mountPath: /etc/ceph/
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: ceph-csi-encryption-kms-config
+              mountPath: /etc/ceph-csi-encryption-kms-config/
+            - name: oidc-token
+              mountPath: /run/secrets/tokens
+              readOnly: true
+        # possibly sidecars like driver‑registrar, liveness, etc.
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+        - name: mountpoint-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: ceph-logdir
+          hostPath:
+            path: /var/log/ceph
+            type: DirectoryOrCreate
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: etc-selinux
+          hostPath:
+            path: /etc/selinux
+            type: DirectoryOrCreate
+        - name: host-mount
+          hostPath:
+            path: /run/mount
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+            type: DirectoryOrCreate
+        - name: ceph-config
+          configMap:
+            name: ceph-config
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config
+        - name: ceph-csi-encryption-kms-config
+          configMap:
+            name: ceph-csi-encryption-kms-config
+        - name: keys-tmp-dir
+          emptyDir:
+            medium: Memory
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-metrics-rbdplugin
+  namespace: ceph-csi
+  labels:
+    app: csi‑metrics
+spec:
+  ports:
+    - name: http‑metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8680
+  selector:
+    app: csi‑rbdplugin
@@ -0,0 +1,2 @@
+#!/usr/bin/bash
+/var/lib/rancher/rke2/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml get --raw /livez
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: rke2-ingress-nginx-controller
+  namespace: kube-system
+  annotations:
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP
+    - name: https
+      port: 443
+      targetPort: https
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: rke2-ingress-nginx
+    loadBalancerIP: 198.18.200.0
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      hostPort:
+        enabled: false
+      service:
+        enabled: true
+        type: LoadBalancer
+        externalTrafficPolicy: Local
+        annotations:
+          purelb.io/service-group: common
@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rancher
+  namespace: cattle-system
+  annotations:
+    kubernetes.io/ingress.class: nginx
+spec:
+  tls:
+    - hosts: [rancher.main.unkin.net]
+      secretName: tls-rancher
+  rules:
+    - host: rancher.main.unkin.net
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: rancher
+                port:
+                  number: 80
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cattle-system
+  labels:
+    kubernetes.io/metadata.name: cattle-system
@@ -0,0 +1,45 @@
+apiVersion: purelb.io/v1
+kind: LBNodeAgent
+metadata:
+  name: common
+  namespace: purelb
+spec:
+  local:
+    extlbint: kube-lb0
+    localint: default
+    sendgarp: false
+---
+apiVersion: purelb.io/v1
+kind: LBNodeAgent
+metadata:
+  name: dmz
+  namespace: purelb
+spec:
+  local:
+    extlbint: kube-lb0
+    localint: default
+    sendgarp: false
+---
+apiVersion: purelb.io/v1
+kind: ServiceGroup
+metadata:
+  name: dmz
+  namespace: purelb
+spec:
+  local:
+    v4pools:
+      - subnet: 198.18.199.0/24
+        pool: 198.18.199.0/24
+        aggregation: /32
+---
+apiVersion: purelb.io/v1
+kind: ServiceGroup
+metadata:
+  name: common
+  namespace: purelb
+spec:
+  local:
+    v4pools:
+      - subnet: 198.18.200.0/24
+        pool: 198.18.200.0/24
+        aggregation: /32
@@ -0,0 +1,9 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-canal
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    flannel:
+      iface: "loopback0"
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'facter/util/helm'
+
+Facter.add(:helm_repos) do
+  confine kernel: 'Linux'
+  confine enc_role: [
+    'roles::infra::k8s::control',
+    'roles::infra::k8s::compute'
+  ]
+
+  setcode do
+    Facter::Util::Helm.get_helm_repos('/usr/bin/helm')
+  end
+end
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'facter'
+require 'json'
+
+# a simple helm module
+module Facter::Util::Helm
+  def self.get_helm_repos(helm_cmd)
+    return [] unless File.executable?(helm_cmd)
+
+    output = Facter::Core::Execution.execute(
+      "#{helm_cmd} repo list --output json --repository-config /etc/helm/repositories.yaml",
+      on_fail: nil
+    )
+    return [] if output.to_s.strip.empty?
+
+    parse_helm_output(output)
+  rescue StandardError => e
+    Facter.debug("helm_repos fact error: #{e}")
+    []
+  end
+
+  def self.parse_helm_output(output)
+    JSON.parse(output).map do |repo|
+      {
+        'name' => repo['name'],
+        'url' => repo['url']
+      }
+    end
+  end
+end
@@ -0,0 +1,121 @@
+# config rke2
+class rke2::config (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+  Hash                    $config_hash = $rke2::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
+  String                  $node_token = $rke2::node_token,
+  Array[String[1]]        $extra_config_files = $rke2::extra_config_files,
+  Boolean                 $csi_ceph_enable = $rke2::csi_ceph_enable,
+  Array[String]           $csi_ceph_files = $rke2::csi_ceph_files,
+  Array[String]           $csi_ceph_templates = $rke2::csi_ceph_templates,
+  Optional[String[1]]     $csi_ceph_key = $rke2::csi_ceph_key,
+  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::csi_ceph_clusterid,
+  Optional[Array[String]] $csi_ceph_monitors = $rke2::csi_ceph_monitors,
+  Optional[String[1]]     $csi_ceph_poolname = $rke2::csi_ceph_poolname,
+){
+
+  # if its not the bootstrap node, add join path to config
+  if $node_type == 'server' {
+    if $trusted['certname'] != $bootstrap_node {
+      $config = merge($config_hash, {
+        server => $join_url,
+        token  => $node_token,
+      } )
+    }else{
+      $config = merge($config_hash, {
+        token  => $node_token,
+      } )
+    }
+  } elsif $node_type == 'agent' {
+      $config = merge($config_hash, {
+        server => $join_url,
+        token  => $node_token,
+      } )
+  }else{
+    $config = $config_hash
+  }
+
+  # create the config file
+  file { $config_file:
+    ensure  => file,
+    content => Sensitive($config.to_yaml),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => Package["rke2-${node_type}"],
+    before  => Service["rke2-${node_type}"],
+  }
+
+  # create a script to verify k8s api is up (used by consul)
+  file {'/usr/local/bin/check_k8s_api.sh':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/rke2/check_k8s_api.sh',
+  }
+
+  # symlink kubectl to path
+  file {'/usr/bin/kubectl':
+    ensure  => link,
+    target  => '/var/lib/rancher/rke2/bin/kubectl',
+    require => Package["rke2-${node_type}"],
+  }
+
+  # when ProtectKernelDefaults=true
+  sysctl { 'vm.overcommit_memory':
+    value  => '1',
+    before => Service["rke2-${node_type}"],
+  }
+  sysctl { 'kernel.panic':
+    value  => '10',
+    before => Service["rke2-${node_type}"],
+  }
+
+  # on the controller nodes only
+  if $node_type == 'server' {
+
+    # manage extra config config
+    $extra_config_files.each |$file| {
+
+      file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
+        ensure  => file,
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0644',
+        source  => "puppet:///modules/rke2/${file}.yaml",
+        require => Service['rke2-server'],
+      }
+    }
+
+    # manage ceph files
+    if $csi_ceph_enable {
+
+      $csi_ceph_files.each |$file| {
+
+        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
+          ensure  => file,
+          owner   => 'root',
+          group   => 'root',
+          mode    => '0644',
+          source  => "puppet:///modules/rke2/${file}.yaml",
+          require => Service['rke2-server'],
+        }
+      }
+
+      $csi_ceph_templates.each |$file| {
+
+        file {"/var/lib/rancher/rke2/server/manifests/${file}.yaml":
+          ensure  => file,
+          owner   => 'root',
+          group   => 'root',
+          mode    => '0644',
+          content => template("rke2/${file}.yaml.erb"),
+          require => Service['rke2-server'],
+        }
+      }
+    }
+  }
+}
@@ -0,0 +1,80 @@
+# manage helm
+class rke2::helm (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
+  Boolean                 $helm_install = $rke2::helm_install,
+  Hash                    $helm_repos = $rke2::helm_repos
+){
+
+  # when installing helm, manage the repos
+  if $helm_install {
+
+    package {'helm':
+      ensure => installed,
+    }
+
+    file { '/etc/helm':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0755',
+    }
+
+    # on the controller nodes only
+    if $node_type == 'server' {
+
+      # check if the repo already exists
+      $helm_repos.each | String $repo, Stdlib::HTTPSUrl $url | {
+
+        # if repo isnt in repo list from helm, install it
+        if ! $facts['helm_repos'].any |$existing| { $existing['name'] == $repo } {
+
+          exec { "helm_add_repo_${repo}":
+            command     => "helm repo add ${repo} ${url} --repository-config /etc/helm/repositories.yaml",
+            path        => ['/usr/bin'],
+            environment => [
+              'KUBECONFIG=/etc/rancher/rke2/rke2.yaml',
+            ],
+          }
+        }
+      }
+
+      # install specific helm charts to bootstrap environment
+      $plb_cmd = 'helm install purelb purelb/purelb \
+                  --create-namespace \
+                  --namespace=purelb \
+                  --repository-config /etc/helm/repositories.yaml'
+      exec { 'install_purelb':
+        command     => $plb_cmd,
+        path        => ['/usr/bin', '/bin'],
+        environment => ['KUBECONFIG=/etc/rancher/rke2/rke2.yaml'],
+        unless      => 'helm list -n purelb | grep -q ^purelb',
+      }
+
+      $cm_cmd = 'helm install cert-manager jetstack/cert-manager \
+                  --namespace cert-manager \
+                  --create-namespace \
+                  --set crds.enabled=true \
+                  --repository-config /etc/helm/repositories.yaml'
+      exec { 'install_cert_manager':
+        command     => $cm_cmd,
+        path        => ['/usr/bin', '/bin'],
+        environment => ['KUBECONFIG=/etc/rancher/rke2/rke2.yaml'],
+        unless      => 'helm list -n cert-manager | grep -q ^cert-manager',
+      }
+
+      $r_cmd = 'helm install rancher rancher-stable/rancher \
+                  --namespace cattle-system \
+                  --set hostname=rancher.main.unkin.net \
+                  --set bootstrapPassword=admin \
+                  --set ingress.tls.source=secret \
+                  --repository-config /etc/helm/repositories.yaml'
+      exec { 'install_rancher':
+        command     => $r_cmd,
+        path        => ['/usr/bin', '/bin'],
+        environment => ['KUBECONFIG=/etc/rancher/rke2/rke2.yaml'],
+        unless      => 'helm list -n cattle-system | grep -q ^rancher',
+      }
+    }
+  }
+}
@@ -0,0 +1,30 @@
+# manage rke2
+class rke2 (
+  Enum['server', 'agent'] $node_type = $rke2::params::node_type,
+  String                  $rke2_version = $rke2::params::rke2_version,
+  String                  $rke2_release = $rke2::params::rke2_release,
+  Stdlib::Absolutepath    $config_file = $rke2::params::config_file,
+  Hash                    $config_hash = $rke2::params::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::params::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::params::bootstrap_node,
+  String                  $node_token = $rke2::params::node_token,
+  Boolean                 $helm_install = $rke2::params::helm_install,
+  Hash                    $helm_repos = $rke2::params::helm_repos,
+  Array[String[1]]        $extra_config_files = $rke2::params::extra_config_files,
+  Stdlib::HTTPUrl         $container_archive_source = $rke2::params::container_archive_source,
+  Boolean                 $csi_ceph_enable = $rke2::params::csi_ceph_enable,
+  Array[String]           $csi_ceph_files = $rke2::params::csi_ceph_files,
+  Array[String]           $csi_ceph_templates = $rke2::params::csi_ceph_templates,
+  Optional[String[1]]     $csi_ceph_key = $rke2::params::csi_ceph_key,
+  Optional[String[1] ]    $csi_ceph_clusterid = $rke2::params::csi_ceph_clusterid,
+  Optional[Array[String]] $csi_ceph_monitors = $rke2::params::csi_ceph_monitors,
+  Optional[String[1]]     $csi_ceph_poolname = $rke2::params::csi_ceph_poolname,
+) inherits rke2::params {
+
+  include rke2::install
+  include rke2::config
+  include rke2::service
+  include rke2::helm
+
+  Class['rke2::install'] ->  Class['rke2::service'] -> Class['rke2::helm']
+}
@@ -0,0 +1,53 @@
+# install rke2
+class rke2::install (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  String                  $rke2_version = $rke2::rke2_version,
+  String                  $rke2_release = $rke2::rke2_release,
+  Stdlib::HTTPUrl         $container_archive_source = $rke2::container_archive_source,
+){
+
+  # versionlock rke2
+  yum::versionlock{"rke2-${node_type}":
+    ensure  => present,
+    version => "${rke2_version}~${rke2_release}",
+  }
+
+  # install rke2
+  package {"rke2-${node_type}":
+    ensure => "${rke2_version}~${rke2_release}",
+  }
+
+  # ensure images path exists
+  file { ['/var/lib/rancher/rke2/agent', '/var/lib/rancher/rke2/agent/images']:
+    ensure  => 'directory',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0750',
+    require => Package["rke2-${node_type}"],
+    before  => Service["rke2-${node_type}"],
+  }
+
+  # download required archive of containers
+  archive { '/var/lib/rancher/rke2/agent/images/rke2-images.linux-amd64.tar.zst':
+    ensure  => present,
+    source  => "https://github.com/rancher/rke2/releases/download/v${rke2_version}%2B${rke2_release}/rke2-images.linux-amd64.tar.zst",
+    require => [
+      Package["rke2-${node_type}"],
+      File['/var/lib/rancher/rke2/agent/images'],
+    ],
+    before  => Service["rke2-${node_type}"],
+  }
+
+  # ensure the images cache file exists
+  file {'/var/lib/rancher/rke2/agent/images/.cache.json':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => [
+      Package["rke2-${node_type}"],
+      File['/var/lib/rancher/rke2/agent/images'],
+    ],
+    before  => Service["rke2-${node_type}"],
+  }
+}
@@ -0,0 +1,22 @@
+# rke2 params
+class rke2::params (
+  Enum['server', 'agent'] $node_type = 'agent',
+  String                  $rke2_version = '1.33.4',
+  String                  $rke2_release = 'rke2r1',
+  Stdlib::Absolutepath    $config_file = '/etc/rancher/rke2/config.yaml',
+  Hash                    $config_hash = {},
+  Stdlib::HTTPSUrl        $join_url = 'https://127.0.0.1:9345',
+  Stdlib::Fqdn            $bootstrap_node = 'localhost.localdomain',
+  String                  $node_token = '',
+  Boolean                 $helm_install = false,
+  Hash                    $helm_repos = {},
+  Array[String[1]]        $extra_config_files = [],
+  Stdlib::HTTPUrl         $container_archive_source = 'https://github.com/rancher/rke2/releases/download',
+  Boolean                 $csi_ceph_enable = false,
+  Array[String]           $csi_ceph_files = [],
+  Array[String]           $csi_ceph_templates = [],
+  Optional[String[1]]     $csi_ceph_key = undef,
+  Optional[String[1] ]    $csi_ceph_clusterid = undef,
+  Optional[Array[String]] $csi_ceph_monitors = undef,
+  Optional[String[1]]     $csi_ceph_poolname = undef,
+) {}
@@ -0,0 +1,13 @@
+# manage rke2 service
+class rke2::service (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+){
+
+  service {"rke2-${node_type}":
+    ensure    => true,
+    enable    => true,
+    subscribe => File[$config_file],
+  }
+
+}
@@ -0,0 +1,65 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ceph-csi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-csi-config
+  namespace: ceph-csi
+data:
+  config.json: |-
+    [
+      {
+        "clusterID": "<%= @csi_ceph_clusterid %>",
+        "monitors": [
+        <% @csi_ceph_monitors.each_with_index do |mon, index| -%>
+          "<%= mon %>"<% if index < @csi_ceph_monitors.length - 1 %>,<% end %>
+        <% end -%>
+        ]
+      }
+    ]
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-csi-encryption-kms-config
+  namespace: ceph-csi
+data:
+  config.json: |-
+    {}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-config
+  namespace: ceph-csi
+data:
+  ceph.conf: |
+    [global]
+    auth_cluster_required = cephx
+    auth_service_required = cephx
+    auth_client_required = cephx
+  keyring: |
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-rbd-sc
+provisioner: rbd.csi.ceph.com
+parameters:
+  clusterID: <%= @csi_ceph_clusterid %>
+  pool: <%= @csi_ceph_poolname %>
+  imageFeatures: layering
+  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi
+  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
+  csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi
+  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
+  csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+mountOptions:
+  - discard
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: csi-rbd-secret
+  namespace: ceph-csi
+stringData:
+  userID: kubernetes
+  userKey: <%= @csi_ceph_key %>
+
@@ -0,0 +1,36 @@
+class profiles::ceph::conf (
+  Hash $config = {}
+) {
+
+  package {[
+    'ceph',
+    'ceph-common'
+  ]:
+    ensure => installed,
+  }
+
+  file {'/etc/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file {'/var/log/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file { '/etc/ceph/ceph.conf':
+    ensure  => file,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0644',
+    content => template('profiles/ceph/conf.erb'),
+    require => Package['ceph-common'],
+  }
+}
@@ -0,0 +1,20 @@
+class profiles::ceph::dashboard (
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
+){
+
+  # export haproxy balancemember
+  profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_9443":
+    service   => 'be_ceph_dashboard',
+    ipaddress => $ipaddress,
+    ports     => [9443],
+    options   => [
+      "cookie ${facts['networking']['hostname']}",
+      'ssl',
+      'verify none',
+      'check',
+      'inter 2s',
+      'rise 3',
+      'fall 2',
+    ]
+  }
+}
@@ -0,0 +1,41 @@
+class profiles::ceph::rgw (
+  Boolean $enable = true,
+  Hash[String, String] $ceph_client_keys = {},
+  Stdlib::Absolutepath $base_path = '/var/lib/ceph'
+){
+
+  $key = $ceph_client_keys[$facts['networking']['hostname']]
+
+  if $enable {
+
+    include profiles::ceph::conf
+
+    package {'ceph-radosgw':
+      ensure => installed,
+    }
+
+    file { [
+      "${base_path}/radosgw",
+      "${base_path}/radosgw/ceph-${facts['networking']['hostname']}"
+    ]:
+      ensure => directory,
+      owner  => 'ceph',
+      group  => 'ceph',
+      mode   => '0750',
+    }
+
+    file { "${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring":
+      ensure  => file,
+      owner   => 'ceph',
+      group   => 'ceph',
+      mode    => '0750',
+      content => Sensitive("[client.${facts['networking']['hostname']}]\n	key = ${key}\n")
+    }
+
+    service {"ceph-radosgw@${facts['networking']['hostname']}":
+      ensure    => true,
+      enable    => true,
+      subscribe => File["${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring"]
+    }
+  }
+}
@@ -11,6 +11,7 @@ class profiles::consul::client (
  Stdlib::Absolutepath $data_dir           = '/opt/consul',
  Array[Hash]          $node_rules         = [],
  Hash                 $ports              = {},
+  Stdlib::IP::Address  $host_addr          = $facts['networking']['ip'],
 ) {

  if $facts['enc_role'] != $members_role {
@@ -42,8 +43,8 @@ class profiles::consul::client (
        'log_level'            => 'INFO',
        'node_name'            => $facts['networking']['fqdn'],
        'retry_join'           => $servers_array,
-        'bind_addr'            => $::facts['networking']['ip'],
-        'advertise_addr'       => $::facts['networking']['ip'],
+        'bind_addr'            => $host_addr,
+        'advertise_addr'       => $host_addr,
        'enable_script_checks' => true,
        'ports'                => $ports,
        'acl'                  => {
@@ -47,7 +47,7 @@ class profiles::dns::base (
  $facts['networking']['interfaces'].each | $interface, $data | {

    # exclude those without ipv4 address, lo, docker0 and anycast addresses
-    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ {
+    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ and $interface !~ /^cilium_/  {

      # use defaults for the primary_interface
      if $interface == $primary_interface {
@@ -3,6 +3,7 @@ define profiles::haproxy::balancemember (
  String              $service,
  Array[Stdlib::Port] $ports,
  Array               $options = ['check'],
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
 ) {

  $location_environment = "${facts['country']}-${facts['region']}-${facts['environment']}"
@@ -12,7 +13,7 @@ define profiles::haproxy::balancemember (
    listening_service => $service,
    ports             => $ports,
    server_names      => $facts['networking']['hostname'],
-    ipaddresses       => $facts['networking']['ip'],
+    ipaddresses       => $ipaddress,
    options           => $options,
    tag               => $balancemember_tag,
  }
@@ -11,6 +11,7 @@ class profiles::metrics::grafana (
  String    $db_pass     = fqdn_rand_string(16),
  Stdlib::Host $db_host  = '127.0.0.1',
  Stdlib::Port $db_port  = 5432,
+  Hash      $plugins     = {}
 ) {

  # set the fqdn
@@ -108,6 +109,7 @@ class profiles::metrics::grafana (
  class { 'grafana':
    cfg      => $cfg,
    ldap_cfg => $ldap_cfg,
+    plugins  => $plugins,
  }

  # fix the package provided systemd service
@@ -0,0 +1,9 @@
+# Managed by Puppet in profiles::ceph::conf
+<% @config.each do |section, settings| -%>
+[<%= section %>]
+<% settings.each do |key, value| -%>
+<%# Convert booleans and numbers to strings, leave strings untouched %>
+<%= key %> = <%= value.is_a?(TrueClass) ? 'true' : value.is_a?(FalseClass) ? 'false' : value %>
+<% end -%>
+
+<% end -%>
@@ -1,33 +1,72 @@
 #!/usr/bin/env bash
+set -euo pipefail
+
+: "${DRY_RUN:=}"   # set DRY_RUN=1 to preview deletions
+
+log() { printf '[%(%F %T)T] %s\n' -1 "$*" >&2; }

 # Function to create symlink for snapshots
 create_symlink() {
-  local osname="$1"
-  local release="$2"
-  local repository="$3"
-  local basepath="$4"
-  local label="$5" # 'monthly', 'weekly', or 'daily'
-  local date_format="$6" # Date format for finding the snapshot
+  local osname="$1"     # e.g. almalinux
+  local release="$2"    # e.g. 9.6
+  local repository="$3" # e.g. appstream
+  local basepath="$4"   # e.g. /shared/apps/packagerepo
+  local label="$5"      # monthly|weekly|daily
+  local date_value="$6" # YYYYMMDD for the snapshot to promote

-  # The path where snapshots are stored
-  local snap_path="${basepath}/snap/${osname}/${release}/${repository}-${date_format}"
-
-  # The target path for the symlink
+  local snap_path="${basepath}/snap/${osname}/${release}/${repository}-${date_value}"
  local symlink_target="${basepath}/snap/${osname}/${release}/${repository}-${label}"

-  # Check if the source directory exists
  if [[ -d "$snap_path" ]]; then
-    # Create the symlink, overwrite if it already exists
    ln -sfn "$snap_path" "$symlink_target"
-    echo "Symlink created for $snap_path -> $symlink_target"
+    log "Symlink set: ${symlink_target} -> ${snap_path}"
+    return 0
  else
-    echo "Snapshot path does not exist: $snap_path"
+    log "Snapshot path does not exist: $snap_path"
    return 1
  fi
 }

+# Cleanup snapshot directories older than a cutoff date (YYYYMMDD)
+cleanup_older_than_monthly() {
+  local osname="$1"
+  local release="$2"
+  local repository="$3"
+  local basepath="$4"
+  local cutoff_date="$5"  # newly promoted monthly date, YYYYMMDD
+
+  local root="${basepath}/snap/${osname}/${release}"
+  local pattern="${repository}-"[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]
+
+  shopt -s nullglob
+  for dir in "${root}"/${pattern}; do
+    # Skip symlinks and non-directories
+    [[ -L "$dir" ]] && continue
+    [[ -d "$dir" ]] || continue
+
+    # Extract the date suffix
+    local name base datepart
+    name="$(basename -- "$dir")"
+    base="${name%-????????}"     # remove last 8 chars
+    datepart="${name#${base}-}"  # after "<repo>-"
+
+    # Only act on valid YYYYMMDD dates
+    if [[ "$datepart" =~ ^[0-9]{8}$ ]]; then
+      if [[ "$datepart" -lt "$cutoff_date" ]]; then
+        if [[ -n "${DRY_RUN}" ]]; then
+          log "[DRY_RUN] Would remove: $dir"
+        else
+          log "Removing: $dir"
+          rm -rf --one-file-system -- "$dir"
+        fi
+      fi
+    fi
+  done
+  shopt -u nullglob
+}
+
 # Determine which snapshot to promote based on the passed argument
-case "$1" in
+case "${1:-}" in
  monthly)
    promote_date=$(date --date="$(date +%Y%m01) -1 month" +%Y%m%d)
    ;;
@@ -43,11 +82,26 @@ case "$1" in
    ;;
 esac

-# Call the function with appropriate arguments
-# Iterate over the repositories to create symlinks for each
+label="$1"
+
+# Process each repo conf
 for conf in /etc/reposync/conf.d/*.conf; do
+  # shellcheck disable=SC1090
  source "$conf"

-  # Create symlink based on the provided argument
-  create_symlink "$OSNAME" "$RELEASE" "$REPOSITORY" "$BASEPATH" "$1" "$promote_date"
+  # Expecting these vars in the conf: OSNAME, RELEASE, REPOSITORY, BASEPATH
+  : "${OSNAME:?missing OSNAME in $conf}"
+  : "${RELEASE:?missing RELEASE in $conf}"
+  : "${REPOSITORY:?missing REPOSITORY in $conf}"
+  : "${BASEPATH:?missing BASEPATH in $conf}"
+
+  # 1) Promote the symlink for this label
+  if create_symlink "$OSNAME" "$RELEASE" "$REPOSITORY" "$BASEPATH" "$label" "$promote_date"; then
+    # 2) If monthly, clean up older snapshot dirs for this repo
+    if [[ "$label" == "monthly" ]]; then
+      cleanup_older_than_monthly "$OSNAME" "$RELEASE" "$REPOSITORY" "$BASEPATH" "$promote_date"
+    fi
+  else
+    log "Skipping cleanup for ${OSNAME}/${RELEASE}/${REPOSITORY} due to missing ${REPOSITORY}-${promote_date}"
+  fi
 done
@@ -0,0 +1,10 @@
+# a role to deploy the ceph rgw
+class roles::infra::ceph::rgw {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}
Author	SHA1	Message	Date
unkinben	dd7b6b73ed	fix: cattle-system namespace Build / precommit (pull_request) Successful in 4m58s Details - cattle-system namespace is created earlier than helm - leave namespaces.yaml to manage cattle-system namespace (required before installing helm/rancher)	2025-09-20 22:35:35 +10:00
unkinben	4c9204858e	feat: define node-token from puppet (#398 ) - define the token on the bootstrap node too, so node-token is defined for new clusters Reviewed-on: #398	2025-09-20 22:25:56 +10:00
unkinben	571a9b25a7	fix: resolve rke2-server errors (#397 ) - kubectl yaml files must not use underscores - replace unicode hyphen with ascii hyphen Reviewed-on: #397	2025-09-20 18:40:18 +10:00
unkinben	762f415d2d	feat: k8s helm rework (#396 ) - remove helm-generated-yaml, replace with helm execs - template/parameterise ceph csi Reviewed-on: #396	2025-09-20 17:40:41 +10:00
unkinben	4e77fb7ee7	feat: manage rancher, purelb, cert-manager (#395 ) This change will install rancher, purelb and cert-manager, then configure a dmz and common ip pool to be used by loadbalancers. The nginx ingres controller is configured to use 198.18.200.0 (common) and announce the ip from all nodes so that it becomes an anycast ip in ospf. - manage the install of rancher, purelb and cert-manager - add rancher ingress routes - add nginx externalip/loadBalancer Reviewed-on: #395	2025-09-14 20:59:39 +10:00
unkinben	6e4bc9fbc7	feat: adding rke2 (#394 ) - manage rke2 repos - add rke2 module (init, params, install, config, service) - split roles::infra::k8s::node -> control/compute roles - moved common k8s config into k8s.yaml - add bootstrap_node, manage server and token fields in rke2 config - manage install of helm - manage node attributes (from puppet facts) - manage frr exclusions for service/cluster network Reviewed-on: #394	2025-09-14 13:27:49 +10:00
unkinben	012e842d7d	feat: add cleanup to autopromoter (#393 ) - ensure the autopromoter removes hardlinks/replicas for repos older than the current promoted monthly - this is to reduce MDS load for ceph, as hardlinks require memory Reviewed-on: #393	2025-09-13 20:08:32 +10:00
unkinben	98a433d366	feat: mirror rke2 repo for rhel9 (#392 ) - create rhel9 mirrors for rke2 1.33 and common Reviewed-on: #392	2025-09-13 19:49:52 +10:00
unkinben	fcd1b049d6	feat: ensure frr_exporter can read ospf socket (#391 ) - add execute permission to frr socket directory Reviewed-on: #391	2025-09-13 15:08:32 +10:00
unkinben	938a6ac990	feat: update docs for puppet (#390 ) - k8s / metallb / cilium created chaos - broke puppet agent and servers - adding issue/resolution here Reviewed-on: #390	2025-09-13 12:57:44 +10:00
unkinben	0665873dc8	feat: update ospf source for learned routes (#388 ) - enable changing the source address for learned ospf routes - this enables the loopback0 interface to be used as a default src address - ensure k8s nodes use loopback0 as default src - ensure incus nodes use loopback0 as default src Reviewed-on: #388	2025-09-07 16:09:21 +10:00
unkinben	ae4eb3a5eb	fix: set loopback0 as source for consul (#387 ) - fix consul service checks for prodnxsr0001-0008 - ensure the loopback0 interface whats bound too Reviewed-on: #387	2025-09-07 15:48:27 +10:00
unkinben	65fb52da55	chore: add user for jelly (#385 ) Reviewed-on: #385	2025-09-04 20:09:43 +10:00
unkinben	d97cbfd570	chore: update src ips for arr stack (#384 ) - allow the arr stack to reach prowlarr Reviewed-on: #384	2025-08-31 18:52:47 +10:00
unkinben	8f5d102945	feat: enabling changing ip for consul client (#383 ) - enable ability to set consul client bind/advertise ip Reviewed-on: #383	2025-08-14 22:55:35 +10:00
unkinben	62aade77ff	feat: add ceph-dashboard to haproxy (#382 ) - add profile to export haproxy backend - add new cert for dashboard.ceph.unkin.net - extend balancemember with ipaddress attribute Reviewed-on: #382	2025-08-14 11:06:11 +10:00
unkinben	83bb3e1085	chore: increase client body size for s3 (#381 ) - s3 clients send objects too large for the default body size Reviewed-on: #381	2025-08-13 16:41:39 +10:00
unkinben	92728047e7	feat: add ceph rgw (#380 ) - start managing ceph configuration file - manage ceph-radosgw - merge the ceph::conf and ceph::node profiles - ensure the ceph repos exist - mange nginx frontend and consul service Reviewed-on: #380	2025-08-13 12:33:41 +10:00
unkinben	f4af5e7b64	chore: add rados gateway role (#379 ) - just enough role to deploy some containers Reviewed-on: #379	2025-08-10 19:31:39 +10:00
unkinben	308d97d783	feat: enable plugins for grafana (#378 ) - add method to install plugins for grafana - ensure victoriametrics-logs-datasource is installed Reviewed-on: #378	2025-08-09 17:57:49 +10:00
				`@@ -0,0 +1 @@`
				rke2::node_token: ENC[PKCS7,MIIB2gYJKoZIhvcNAQcDoIIByzCCAccCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAVPo7WjzlJQQhxefhV1bu1/+Jo/LY3gMmVGicDnDloGQd1jbdDtNz9wi6Hqqht1xQPdn22XmvvrVXtPhsdjDCGqWxmfZ0qWQVl1Ju2WIh5WsyyZgdE96k2+y7Cg5Dl0brX2m9YSZfow5BF8J8EnCDdRZncOCtFl/SU8ipPEq2uJJR+Y9sJv6aJflnFLCEYgJNbZY9ljMcs5ssJ21VpIqWYA0Z6vKVqOUeIWWKbYZUIoEml7sj3ktmw3loMYA6ED0/nzyYvRVizTvtGXl3IWGVDSt8rQ/kNhzKqURVOsgwfbt7un4n2Kxkreiydj3R6PbLdkpHdtu25dbjue8HLQkc4zCBnAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQMmIatr2oBei35+evghIkLIBwWT6F1WPDV3PA/QYNusfYcI1KrMRYFlDYyEkg/Tf+0gOpty9rdnYL+MQO2fTU9Br+INTr1oJcxJp/Lqap2+NibmBfZcLUIMn3q1S/jZp9BGQTk6RSwODqQ2x5GxDATinrtiUR4TXIIaiaP3KWlP8A7g==]