feat: adding rke2

- manage rke2 repos - add rke2 module (init, params, install, config, service) - exclude setting ips for cilium interfaces - split roles::infra::k8s::node -> control/compute roles - add bootstrap_node, manage server and token fields in rke2 config
feat: update ospf source for learned routes (#388 )
2025-09-07 16:13:51 +10:00 · 2025-09-07 16:09:21 +10:00 · 2025-09-07 15:48:27 +10:00 · 2025-09-04 20:09:43 +10:00 · 2025-08-31 18:52:47 +10:00 · 2025-08-14 22:55:35 +10:00
31 changed files with 645 additions and 50 deletions
@@ -368,6 +368,28 @@ profiles::ceph::client::mons:
  - 10.18.15.1
  - 10.18.15.2
  - 10.18.15.3
+
+profiles::ceph::conf::config:
+  global:
+    auth_client_required: 'cephx'
+    auth_cluster_required: 'cephx'
+    auth_service_required: 'cephx'
+    fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+    mon_allow_pool_delete: true
+    mon_initial_members: 'prodnxsr0009,prodnxsr0010,prodnxsr0011,prodnxsr0012,prodnxsr0013'
+    mon_host: '198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13'
+    ms_bind_ipv4: true
+    ms_bind_ipv6: false
+    osd_crush_chooseleaf_type: 1
+    osd_pool_default_min_size: 2
+    osd_pool_default_size: 3
+    osd_pool_default_pg_num: 128
+    public_network: >
+      198.18.23.1/32,198.18.23.2/32,198.18.23.3/32,198.18.23.4/32,
+      198.18.23.5/32,198.18.23.6/32,198.18.23.7/32,198.18.23.8/32,
+      198.18.23.9/32,198.18.23.10/32,198.18.23.11/32,198.18.23.12/32,
+      198.18.23.13/32
+
 #profiles::base::hosts::additional_hosts:
 #  - ip: 198.18.17.9
 #    hostname: prodinf01n09.main.unkin.net
@@ -10,6 +10,7 @@ profiles::haproxy::dns::vrrp_cnames:
  - git.unkin.net
  - fafflix.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net

 profiles::haproxy::mappings:
  fe_http:
@@ -27,6 +28,7 @@ profiles::haproxy::mappings:
      - 'fafflix.unkin.net be_jellyfin'
      - 'git.unkin.net be_gitea'
      - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'
  fe_https:
    ensure: present
    mappings:
@@ -42,6 +44,7 @@ profiles::haproxy::mappings:
      - 'fafflix.unkin.net be_jellyfin'
      - 'git.unkin.net be_gitea'
      - 'grafana.unkin.net be_grafana'
+      - 'dashboard.ceph.unkin.net be_ceph_dashboard'

 profiles::haproxy::frontends:
  fe_http:
@@ -62,6 +65,7 @@ profiles::haproxy::frontends:
        - 'acl_fafflix req.hdr(host) -i fafflix.unkin.net'
        - 'acl_gitea req.hdr(host) -i git.unkin.net'
        - 'acl_grafana req.hdr(host) -i grafana.unkin.net'
+        - 'acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net'
        - 'acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24'
      use_backend:
        - "%[req.hdr(host),lower,map(/etc/haproxy/fe_https.map,be_default)]"
@@ -79,6 +83,7 @@ profiles::haproxy::frontends:
        - 'set-header X-Frame-Options DENY if acl_fafflix'
        - 'set-header X-Frame-Options DENY if acl_gitea'
        - 'set-header X-Frame-Options DENY if acl_grafana'
+        - 'set-header X-Frame-Options DENY if acl_ceph_dashboard'
        - 'set-header X-Content-Type-Options nosniff'
        - 'set-header X-XSS-Protection 1;mode=block'

@@ -262,6 +267,26 @@ profiles::haproxy::backends:
      redirect: 'scheme https if !{ ssl_fc }'
      stick-table: 'type ip size 200k expire 30m'
      stick: 'on src'
+  be_ceph_dashboard:
+    description: Backend for Ceph Dashboard from Mgr instances
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-check:
+        - expect status 200
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 9443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+      stick-table: 'type ip size 200k expire 30m'
+      stick: 'on src'

 profiles::haproxy::certlist::enabled: true
 profiles::haproxy::certlist::certificates:
@@ -276,6 +301,7 @@ profiles::haproxy::certlist::certificates:
  - /etc/pki/tls/letsencrypt/fafflix.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/letsencrypt/git.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/letsencrypt/grafana.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/dashboard.ceph.unkin.net/fullchain_combined.pem
  - /etc/pki/tls/vault/certificate.pem

 # additional altnames
@@ -303,3 +329,4 @@ certbot::client::domains:
  - fafflix.unkin.net
  - git.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net
@@ -70,7 +70,7 @@ profiles::nginx::simpleproxy::locations:
  arrstack_web_external:
    location_satisfy: any
    location_allow:
-      - 198.18.13.47
-      - 198.18.13.50
-      - 198.18.13.51
-      - 198.18.13.52
+      - 198.18.26.161
+      - 198.18.27.131
+      - 198.18.28.165
+      - 198.18.29.32
@@ -211,6 +211,18 @@ glauth::users:
    loginshell: '/bin/bash'
    homedir: '/home/waewak'
    passsha256: 'd9bb99634215fe031c3bdca94149a165192fe8384ecaa238a19354c2f760a811'
+  debvin:
+    user_name: 'debvin'
+    givenname: 'Debbie'
+    sn: 'Vincent'
+    mail: 'debvin@users.main.unkin.net'
+    uidnumber: 20009
+    primarygroup: 20000
+    othergroups:
+      - 20010 # jelly
+    loginshell: '/bin/bash'
+    homedir: '/home/debvin'
+    passsha256: 'cdac05ddb02e665d4ea65a974995f38a10236bc158731d92d78f6cde89b294a1'

 glauth::services:
  svc_jellyfin:
@@ -0,0 +1,8 @@
+---
+
+profiles::ceph::rgw::ceph_client_keys:
+  ausyd1nxvm2115: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAovOUUtTm/9nXWj6D+kVhmbZedVVkA5N80ULNYqISrv0A+KlXCx/V2sB56SpJ9eL6aPHdC+CKkrtrrdo1h2j2em7wTA5ghhbFVF0NIS7bbn0jzpl5YqbUrNxOtDSpjzX1aCGnyMvw69Lp+NcHwxIj+1XFgK6335138s9wbX3HmYF3jcsQkMqTzynELe1OQPWFXVKTjUfFFLdCQOFryp8UY8L9j/PpV6wd4w6p7R6eXhX21rjSaN4aqN1zjsnF2OVhL8Ge0QxMhePWKGOqsUfi72kh3II028DjqU0DcZQvoxnoqRPyUUjysH0nTKoLeXOGNgJdphY1dHBJ+SJnw2gqQDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB9wA3ZJjmU95W+9r8va5uagDAMAv2APfdzrPnAU7NJPL+IW08osGuQWWamqF+XGVeHRoBmoFKwZ7grYRV2e3aabyc=]
+  ausyd1nxvm2116: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAf5ksy/pyUZSwTh+HiKw+1Uhj16A0DVZEKAbkKUQzXVmc+QpL4Dn7YBoXlEwrY8CcsTrxTQjADvtu9FC3o34QIdh06noSgYYA+7fna2A+9+oYoNtgwC3b8LeglxO/SQ9dKoJ90jRtmlw5P/CtrxA2RelMK6FNRekp1CaWMM4q20fJGgr/E33vgx38UJyp4/q0bTu2lLehCuDUP80j3XGbSNZ2snfYdIo91Cl+nSxLSU2TdnFpWaabsH19HwDnkWGiILlLBVvvhY7copCxs5DS1ueoOTCsqnWSrTrBMJjnu7WZd/s4NLw/0q/UP5xcFA51caY3Kv+sI6bfIYkNoLazwDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDrCF16nrtukRPamx1VbGY+gDAK5dw0kV8MpATpwxTkJG6JtlFlwdpU9THs1bNwqSRD1ZhEWxQeWwsyyTtjUXi4bP8=]
+  ausyd1nxvm2117: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAKtvsgDk2QTmL8flBBE4nA43sRSnroq4I6T2CUAYv/lRdzCCWvE961o/51zyQEz8L5QyDo7L3gcGYIBqthYjRe9Gp5a5d4qds0qskupgQKnb0KR2RLFVUtH5vxHqyJZHjXaP+PQreyRoSIfRWXAdoZu544FeJ9DKKbmEzZaH5B2OdDMrf4Ufuud0maEAw0PJthS//ghCfGi74F1xlJnIWVvMhp66b0iMxC+ACClEHunG3oKx7M/w05HllG0wcxPTg4PFrbnFXjRuIxsykF9aVHJkRnCdgbMXRM4o6FrYyZRR74F1HKRujFCUA7kYWDKLxHxJpYCvCHp4HMhfzjs824zBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBbaSig6kgVVNfmSI53sNimgDDQ5O10Dzfa7S7RdJVLLUFBaZ5TG2g3Bwmy0k3wKZvABYMuYyOxQdfk6eMsKC+sC5w=]
+  ausyd1nxvm2118: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAdKl0ude3ZxW0ihyA758mpQp43qZvzTI2Bp83WzCp2ifJCgAIjRWdd01P64rSHaa4lus/wqB9jxg38g6DrN4ejX56Y/CJ6GQKxz6b1BO5nDfsLx6QEzCt+cfg5d/PPoTtEpz2VSvDfxFUrHiABA6++Sqzb9Og+nQCFMYJD3NHCk67QpkjPGQ/ejZk4MNXZQVCfKOlFqay/fF0jEmQixFOlX/Fdm9UoKttbrKluUmzpaVUzfGRaTTFVgzc3x2t/z1q1k0P7ClI9Uu02kUXpFVs9LPX99Zc2GtrnP06mYqqARhWF1NMK0zlmxtKpfObahRP/HmtI3fgnQsU1Cpwah0emTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAILqpYx3FKY3xXLJRu2oDlgDCIOXeX6hxpu0qpj5c/9jMUSeV2DIydnxO+MiT3mceS50ip8B+zGQy5UedPmLt36Zs=]
+  ausyd1nxvm2119: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEASlZglUxazp+9azfV3QkgRv+ACo+MH0RO5b18blbelgdmr38iwK7MwwEFpEfVJEyc/ph9RunWwrMmofDQHj5bBribfzZ2pH2CGiOrR0i5lZMtN0yQXPBA/+jm1Pi1AWGJLtoquuhMbibuHOTiXwBCBVrHHHaFTR5Xt34ABN/p/mCaG+N9nWux93msHCCextCalKBMmPhmI2q6HodfjanEVgYAe3/5hRPnpsi6IGSDNGygsTC3MG+hjGMpNF8izbwk9Lpzn6kY51aeNxI2ed9Jm8UZ/k+8b+o7ZQyWIBbf7DTFpEzk4G46puaDbXIorBWQ4azCjN3gt8VB91hwihtzcDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB6U+9z4cSzMTA1z9bmoX82gDBfy5zbRPK8GxImJo6evecMOTtaY2c4aEnESXtGBCS02enmxljv9dv1UYQD0/a6S3A=]
@@ -0,0 +1,60 @@
+---
+hiera_include:
+  - profiles::ceph::rgw
+  - profiles::nginx::simpleproxy
+
+profiles::ceph::rgw::enable: true
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - radosgw.main.unkin.net
+  - radosgw.service.consul
+  - radosgw.query.consul
+  - "radosgw.service.%{facts.country}-%{facts.region}.consul"
+
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+
+# manage a simple nginx reverse proxy
+profiles::nginx::simpleproxy::nginx_vhost: 'radosgw.service.consul'
+profiles::nginx::simpleproxy::nginx_aliases:
+  - radosgw.service.au-syd1.consul
+profiles::nginx::simpleproxy::proxy_port: 7480
+profiles::nginx::simpleproxy::proxy_path: '/'
+nginx::client_max_body_size: 100M
+
+# manage consul service
+consul::services:
+  radosgw:
+    service_name: 'radosgw'
+    address: "%{facts.networking.ip}"
+    port: 443
+    checks:
+      - id: 'radosgw_https_check'
+        name: 'RADOSGW HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: radosgw
+    disposition: write
@@ -6,6 +6,7 @@ hiera_include:
  - zfs
  - profiles::ceph::node
  - profiles::ceph::client
+  - profiles::ceph::dashboard
  - profiles::storage::cephfsvols
  - exporters::frr_exporter

@@ -36,6 +37,7 @@ profiles::ssh::sign::principals:
  - "%{facts.networking.interfaces.enp3s0.ip}"

 # configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
 consul::services:
  incus:
    service_name: 'incus'
@@ -121,6 +123,9 @@ profiles::yum::global::repos:
 # dns
 profiles::dns::base::primary_interface: loopback0

+# dashboard/haproxy
+profiles::ceph::dashboard::ipaddress: "%{hiera('networking_loopback0_ip')}"
+
 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true
@@ -150,12 +155,11 @@ networking::interfaces:
    netmask: 255.255.255.255
    mtu: 1500

-exporters::node_exporter::addr: "%{hiera('networking_loopback0_ip')}"
-
 # frrouting
 exporters::frr_exporter::enable: true
-exporters::frr_exporter::addr: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_redistribute:
  - connected
 frrouting::ospfd_interfaces:
@@ -0,0 +1 @@
+rke2::node_token: ENC[PKCS7,MIIBuQYJKoZIhvcNAQcDoIIBqjCCAaYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWU56ILL1Te14z0vG9Q9tD9DOlRfxasZHuChZlWHiw3MLOv3FwEc9+jEW20zMJCw6xoTITJ6mXN1fXe48fT3hX5jaBijoqLpfWp64zXomXCEyLh+DZn1N8ETRFS/gB6Fk1PjkCv4x4jd92tTI28gQQAJCb5oNTJzeCFJtovECSL2ElpRJKvdkf+fFtTHdpaTcX4oASz2WUyVux8oAwkGUCRlulzpAJhcwCYoKVAq1wiFRaicK/vZ9DJBlN1HrixKNCsKwRYLZYg0Qh0NjjJPI3UlrR6l06qnTvHhYAOg2BvfQIVlcUI5NjOfUD96GqTb8hFoWhsfhQ0GQT385QmsMlDB8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCj5SiNg/nNkDbK8gR+45sMgFCzjCs/vVqJ+dFQA7r7nVhejb/bKiRtWmEnnvDSN0zOVHAueCeImNBfWB3+ZDSPbXvjN6audmdmeMV+9rvq05Nxcsb5Lcawh6DUzkJWazGH3A==]
@@ -1,42 +1,205 @@
 ---
-profiles::pki::vault::alt_names:
-  - k8s-control.service.consul
-  - k8s-control.query.consul
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
+hiera_include:
+  - profiles::selinux::setenforce
+  - frrouting
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - exporters::frr_exporter
+  - rke2

-profiles::ssh::sign::principals:
-  - k8s-control.service.consul
-  - k8s-control.query.consul
-  - "k8s-control.service.%{facts.country}-%{facts.region}.consul"
+# manage rke2
+rke2::bootstrap_node: prodnxsr0001.main.unkin.net
+rke2::join_url: https://join-k8s.service.consul:9345
+rke2::node_type: server
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "join-k8s.service.consul"
+    - "api-k8s.service.consul"
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: cilium

-# configure consul service
-consul::services:
-  k8s-control:
-    service_name: 'k8s-control'
-    tags:
-      - 'k8s'
-      - 'container'
-    address: "%{facts.networking.fqdn}"
-    port: 6443
-    checks:
-      - id: 'k8s-control_https_check'
-        name: 'k8s-control HTTPS Check'
-        http: "https://%{facts.networking.fqdn}:6443"
-        method: 'GET'
-        tls_skip_verify: true
-        interval: '10s'
-        timeout: '1s'
-profiles::consul::client::node_rules:
-  - resource: service
-    segment: k8s-control
-    disposition: write
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
+
+profiles::packages::include:
+  bridge-utils: {}
+  cephadm: {}
+
+profiles::selinux::setenforce::mode: disabled
+
+profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_package: false
+profiles::ceph::client::manage_ceph_paths: false
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+profiles::ceph::client::mons:
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+
+# additional repos
+profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+
+# dns
+profiles::dns::base::primary_interface: loopback0

 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true
 networking::interfaces:
-  eth0:
+  "%{hiera('networking_1000_iface')}":
    type: physical
+    ipaddress: "%{hiera('networking_1000_ip')}"
+    gateway: 198.18.15.254
+    txqueuelen: 10000
    forwarding: true
-    dhcp: true
+  "%{hiera('networking_2500_iface')}":
+    type: physical
+    ipaddress: "%{hiera('networking_2500_ip')}"
    mtu: 1500
+    txqueuelen: 10000
+    forwarding: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback1:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback1_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+  loopback2:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback2_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+consul::services:
+  api-k8s:
+    service_name: 'api-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 6443
+    checks:
+      - id: 'api-k8s_livez_check'
+        name: 'api-k8s livez Check'
+        args:
+          - sudo
+          - /usr/local/bin/check_k8s_api.sh
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'rke2_tcp_check_9345'
+        name: 'rke2 TCP Check 9345'
+        tcp: "%{hiera('networking_loopback0_ip')}:9345"
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
+    disposition: write
+  - resource: service
+    segment: frr_exporter
+    disposition: write
+
+# frrouting
+exporters::frr_exporter::enable: true
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  "%{hiera('networking_1000_iface')}":
+    area: 0.0.0.0
+  "%{hiera('networking_2500_iface')}":
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+  loopback1:
+    area: 0.0.0.0
+  loopback2:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# add loopback interfaces to ssh list
+ssh::server::options:
+  ListenAddress:
+    - "%{hiera('networking_loopback0_ip')}"
+    - "%{hiera('networking_1000_ip')}"
+    - "%{hiera('networking_2500_ip')}"
+
+profiles::ssh::sign::principals:
+  - "%{hiera('networking_loopback0_ip')}"
+  - "%{hiera('networking_1000_ip')}"
+  - "%{hiera('networking_2500_ip')}"
+
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"
+
+sudo::configs:
+  consul-checks:
+    priority: 20
+    content: |
+      consul ALL=(ALL) NOPASSWD: /usr/local/bin/check_k8s_api.sh
@@ -5,6 +5,24 @@ hiera_include:
  - profiles::ceph::node
  - profiles::ceph::client
  - exporters::frr_exporter
+  - profiles::rke2::node
+
+# manage rke2
+profiles::rke2::node::servers:
+  - prodnxsr0001.main.unkin.net
+  - prodnxsr0002.main.unkin.net
+  - prodnxsr0003.main.unkin.net
+
+rke2::config_hash:
+  bind-address: "%{hiera('networking_loopback0_ip')}"
+  advertise-address: "%{hiera('networking_loopback0_ip')}"
+  node-ip: "%{hiera('networking_loopback0_ip')}"
+  node-external-ip: "%{hiera('networking_loopback0_ip')}"
+  cluster-domain: "svc.k8s.unkin.net"
+  tls-san:
+    - "api.k8s.unkin.net"
+    - "join.k8s.unkin.net"
+  cni: cilium

 # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
 python::manage_dev_package: false
@@ -25,6 +43,7 @@ profiles::ceph::client::mons:
  - 198.18.23.11
  - 198.18.23.12
  - 198.18.23.13
+
 # additional repos
 profiles::yum::global::repos:
  ceph:
@@ -55,6 +74,20 @@ profiles::yum::global::repos:
    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
    mirrorlist: absent
+  rancher-rke2-common-latest:
+    name: rancher-rke2-common-latest
+    descr: rancher-rke2-common-latest
+    target: /etc/yum.repos.d/rke2-common.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/common/centos/%{facts.os.release.major}/noarch
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent
+  rancher-rke2-1-33-latest:
+    name: rancher-rke2-1-33-latest
+    descr: rancher-rke2-1-33-latest
+    target: /etc/yum.repos.d/rke2-1-33.repo
+    baseurl: https://rpm.rancher.io/rke2/latest/1.33/centos/%{facts.os.release.major}/x86_64
+    gpgkey: https://rpm.rancher.io/public.key
+    mirrorlist: absent

 # dns
 profiles::dns::base::primary_interface: loopback0
@@ -91,18 +124,47 @@ networking::interfaces:
    netmask: 255.255.255.255
    mtu: 1500

-# consul
+# configure consul service
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+consul::services:
+  api-k8s:
+    service_name: 'api-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 6443
+    checks:
+      - id: 'api-k8s_https_check'
+        name: 'api-k8s HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:6443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+  join-k8s:
+    service_name: 'join-k8s'
+    address: "%{facts.networking.fqdn}"
+    port: 9345
+    checks:
+      - id: 'etcd_tcp_check_9345'
+        name: 'ETCD TCP Check 9345'
+        tcp: "%{facts.networking.fqdn}:9345"
+        interval: '10s'
+        timeout: '1s'
 profiles::consul::client::node_rules:
+  - resource: service
+    segment: api-k8s
+    disposition: write
+  - resource: service
+    segment: join-k8s
+    disposition: write
  - resource: service
    segment: frr_exporter
    disposition: write

-exporters::node_exporter::addr: "%{hiera('networking_loopback0_ip')}"
-
 # frrouting
 exporters::frr_exporter::enable: true
-exporters::frr_exporter::addr: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"
 frrouting::ospfd_redistribute:
  - connected
 frrouting::ospfd_interfaces:
@@ -130,3 +192,8 @@ profiles::ssh::sign::principals:
  - "%{hiera('networking_loopback0_ip')}"
  - "%{hiera('networking_1000_ip')}"
  - "%{hiera('networking_2500_ip')}"
+
+profiles::pki::vault::alt_names:
+  - api-k8s.service.consul
+  - api-k8s.query.consul
+  - "api-k8s.service.%{facts.country}-%{facts.region}.consul"
@@ -11,6 +11,9 @@ profiles::metrics::grafana::db_name: "%{hiera('profiles::sql::postgresdb::dbname
 profiles::metrics::grafana::db_user: "%{hiera('profiles::sql::postgresdb::dbuser')}"
 profiles::metrics::grafana::db_pass: "%{hiera('profiles::sql::postgresdb::dbpass')}"
 profiles::metrics::grafana::pgsql_backend: true
+profiles::metrics::grafana::plugins:
+  victoriametrics-logs-datasource:
+    ensure: present

 # additional altnames
 profiles::pki::vault::alt_names:
@@ -15,3 +15,4 @@ certbot::domains:
  - fafflix.unkin.net
  - git.unkin.net
  - grafana.unkin.net
+  - dashboard.ceph.unkin.net
@@ -4,7 +4,6 @@ class exporters::frr_exporter (
  String  $group          = 'frr_exporter',
  Boolean $manage_user    = true,
  Boolean $manage_service = true,
-  Stdlib::Host $addr      = $facts['networking']['ip'],
  Stdlib::Port $port      = 9342,
  Stdlib::Absolutepath $exec_path  = '/usr/bin/frr_exporter',
  Stdlib::Absolutepath $socket_dir = '/var/run/frr',
@@ -46,7 +45,7 @@ class exporters::frr_exporter (
    # manage consul service
    consul::service { 'frr_exporter':
      service_name => 'frr_exporter',
-      address      => $addr,
+      address      => $facts['networking']['ip'],
      port         => $port,
      tags         => [
        'metrics',
@@ -4,7 +4,6 @@ class exporters::node_exporter (
  String  $group          = 'node_exporter',
  Boolean $manage_user    = true,
  Boolean $manage_service = true,
-  Stdlib::Host $addr      = $facts['networking']['ip'],
  Stdlib::Port $port      = 9100,
  Stdlib::Absolutepath $exec_path     = '/usr/bin/node_exporter',
  Boolean $cleanup_old_node_exporter  = false,
@@ -59,7 +58,7 @@ class exporters::node_exporter (
    # manage consul service
    consul::service { 'node_exporter':
      service_name => 'node_exporter',
-      address      => $addr,
+      address      => $facts['networking']['ip'],
      port         => $port,
      tags         => [
        'metrics',
@@ -14,6 +14,8 @@ class frrouting (
  Optional[String] $mpls_ldp_router_id       = undef,
  Optional[String] $mpls_ldp_transport_addr  = undef,
  Array[String] $mpls_ldp_interfaces         = [],
+  Boolean $ospf_preferred_source_enable      = false,
+  Optional[Stdlib::IP::Address] $ospf_preferred_source = undef,
 ) {

  $daemons_defaults = {
@@ -43,3 +43,11 @@ mpls ldp
 exit-address-family
 exit
 <% end -%>
+<% if @ospf_preferred_source_enable && @ospf_preferred_source -%>
+ip prefix-list ANY seq 5 permit 0.0.0.0/0 le 32
+route-map OSPF-SRC permit 10
+ match ip address prefix-list ANY
+ set src <%= @ospf_preferred_source %>
+exit
+ip protocol ospf route-map OSPF-SRC
+<% end -%>
@@ -0,0 +1,2 @@
+#!/usr/bin/bash
+/var/lib/rancher/rke2/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml get --raw /livez
@@ -0,0 +1,49 @@
+# config rke2
+class rke2::config (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+  Hash                    $config_hash = $rke2::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
+  String                  $node_token = $rke2::node_token,
+){
+
+  # if its not the bootstrap node, add join path to config
+  if $node_type == 'server' {
+    if $trusted['certname'] != $bootstrap_node {
+      $config = merge($config_hash, {
+        server => $join_url,
+        token  => $node_token,
+      } )
+    }else{
+      $config = $config_hash
+    }
+  }else{
+    $config = $config_hash
+  }
+
+  # create the config file
+  file { $config_file:
+    ensure  => file,
+    content => Sensitive($config.to_yaml),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }
+
+  # create a script to verify k8s api is up (used by consul)
+  file {'/usr/local/bin/check_k8s_api.sh':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/rke2/check_k8s_api.sh'
+  }
+
+  # symlink kubectl to path
+  file {'/usr/bin/kubectl':
+    ensure => link,
+    target => '/var/lib/rancher/rke2/bin/kubectl',
+  }
+
+}
@@ -0,0 +1,16 @@
+# manage rke2
+class rke2 (
+  Enum['server', 'agent'] $node_type = $rke2::params::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::params::config_file,
+  Hash                    $config_hash = $rke2::params::config_hash,
+  Stdlib::HTTPSUrl        $join_url = $rke2::params::join_url,
+  Stdlib::Fqdn            $bootstrap_node = $rke2::params::bootstrap_node,
+  String                  $node_token = $rke2::params::node_token,
+) inherits rke2::params {
+
+  include rke2::install
+  include rke2::config
+  include rke2::service
+
+  Class['rke2::install'] -> Class['rke2::config'] -> Class['rke2::service']
+}
@@ -0,0 +1,10 @@
+# install rke2
+class rke2::install (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+){
+
+  package {"rke2-${node_type}":
+    ensure => installed,
+  }
+
+}
@@ -0,0 +1,9 @@
+# rke2 params
+class rke2::params (
+  Enum['server', 'agent'] $node_type = 'agent',
+  Stdlib::Absolutepath    $config_file = '/etc/rancher/rke2/config.yaml',
+  Hash                    $config_hash = {},
+  Stdlib::HTTPSUrl        $join_url = 'https://127.0.0.1:9345',
+  Stdlib::Fqdn            $bootstrap_node = 'localhost.localdomain',
+  String                  $node_token = '',
+) {}
@@ -0,0 +1,13 @@
+# manage rke2 service
+class rke2::service (
+  Enum['server', 'agent'] $node_type = $rke2::node_type,
+  Stdlib::Absolutepath    $config_file = $rke2::config_file,
+){
+
+  service {"rke2-${node_type}":
+    ensure    => true,
+    enable    => true,
+    subscribe => File[$config_file],
+  }
+
+}
@@ -0,0 +1,36 @@
+class profiles::ceph::conf (
+  Hash $config = {}
+) {
+
+  package {[
+    'ceph',
+    'ceph-common'
+  ]:
+    ensure => installed,
+  }
+
+  file {'/etc/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file {'/var/log/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file { '/etc/ceph/ceph.conf':
+    ensure  => file,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0644',
+    content => template('profiles/ceph/conf.erb'),
+    require => Package['ceph-common'],
+  }
+}
@@ -0,0 +1,20 @@
+class profiles::ceph::dashboard (
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
+){
+
+  # export haproxy balancemember
+  profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_9443":
+    service   => 'be_ceph_dashboard',
+    ipaddress => $ipaddress,
+    ports     => [9443],
+    options   => [
+      "cookie ${facts['networking']['hostname']}",
+      'ssl',
+      'verify none',
+      'check',
+      'inter 2s',
+      'rise 3',
+      'fall 2',
+    ]
+  }
+}
@@ -0,0 +1,41 @@
+class profiles::ceph::rgw (
+  Boolean $enable = true,
+  Hash[String, String] $ceph_client_keys = {},
+  Stdlib::Absolutepath $base_path = '/var/lib/ceph'
+){
+
+  $key = $ceph_client_keys[$facts['networking']['hostname']]
+
+  if $enable {
+
+    include profiles::ceph::conf
+
+    package {'ceph-radosgw':
+      ensure => installed,
+    }
+
+    file { [
+      "${base_path}/radosgw",
+      "${base_path}/radosgw/ceph-${facts['networking']['hostname']}"
+    ]:
+      ensure => directory,
+      owner  => 'ceph',
+      group  => 'ceph',
+      mode   => '0750',
+    }
+
+    file { "${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring":
+      ensure  => file,
+      owner   => 'ceph',
+      group   => 'ceph',
+      mode    => '0750',
+      content => Sensitive("[client.${facts['networking']['hostname']}]\n	key = ${key}\n")
+    }
+
+    service {"ceph-radosgw@${facts['networking']['hostname']}":
+      ensure    => true,
+      enable    => true,
+      subscribe => File["${base_path}/radosgw/ceph-${facts['networking']['hostname']}/keyring"]
+    }
+  }
+}
@@ -11,6 +11,7 @@ class profiles::consul::client (
  Stdlib::Absolutepath $data_dir           = '/opt/consul',
  Array[Hash]          $node_rules         = [],
  Hash                 $ports              = {},
+  Stdlib::IP::Address  $host_addr          = $facts['networking']['ip'],
 ) {

  if $facts['enc_role'] != $members_role {
@@ -42,8 +43,8 @@ class profiles::consul::client (
        'log_level'            => 'INFO',
        'node_name'            => $facts['networking']['fqdn'],
        'retry_join'           => $servers_array,
-        'bind_addr'            => $::facts['networking']['ip'],
-        'advertise_addr'       => $::facts['networking']['ip'],
+        'bind_addr'            => $host_addr,
+        'advertise_addr'       => $host_addr,
        'enable_script_checks' => true,
        'ports'                => $ports,
        'acl'                  => {
@@ -47,7 +47,7 @@ class profiles::dns::base (
  $facts['networking']['interfaces'].each | $interface, $data | {

    # exclude those without ipv4 address, lo, docker0 and anycast addresses
-    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ {
+    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ and $interface !~ /^cilium_/  {

      # use defaults for the primary_interface
      if $interface == $primary_interface {
@@ -3,6 +3,7 @@ define profiles::haproxy::balancemember (
  String              $service,
  Array[Stdlib::Port] $ports,
  Array               $options = ['check'],
+  Stdlib::IP::Address $ipaddress = $facts['networking']['ip'],
 ) {

  $location_environment = "${facts['country']}-${facts['region']}-${facts['environment']}"
@@ -12,7 +13,7 @@ define profiles::haproxy::balancemember (
    listening_service => $service,
    ports             => $ports,
    server_names      => $facts['networking']['hostname'],
-    ipaddresses       => $facts['networking']['ip'],
+    ipaddresses       => $ipaddress,
    options           => $options,
    tag               => $balancemember_tag,
  }
@@ -11,6 +11,7 @@ class profiles::metrics::grafana (
  String    $db_pass     = fqdn_rand_string(16),
  Stdlib::Host $db_host  = '127.0.0.1',
  Stdlib::Port $db_port  = 5432,
+  Hash      $plugins     = {}
 ) {

  # set the fqdn
@@ -108,6 +109,7 @@ class profiles::metrics::grafana (
  class { 'grafana':
    cfg      => $cfg,
    ldap_cfg => $ldap_cfg,
+    plugins  => $plugins,
  }

  # fix the package provided systemd service
@@ -0,0 +1,9 @@
+# Managed by Puppet in profiles::ceph::conf
+<% @config.each do |section, settings| -%>
+[<%= section %>]
+<% settings.each do |key, value| -%>
+<%# Convert booleans and numbers to strings, leave strings untouched %>
+<%= key %> = <%= value.is_a?(TrueClass) ? 'true' : value.is_a?(FalseClass) ? 'false' : value %>
+<% end -%>
+
+<% end -%>
@@ -0,0 +1,10 @@
+# a role to deploy the ceph rgw
+class roles::infra::ceph::rgw {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}
Author	SHA1	Message	Date
unkinben	a06b08e78b	feat: adding rke2 - manage rke2 repos - add rke2 module (init, params, install, config, service) - exclude setting ips for cilium interfaces - split roles::infra::k8s::node -> control/compute roles - add bootstrap_node, manage server and token fields in rke2 config	2025-09-07 16:13:51 +10:00
unkinben	0665873dc8	feat: update ospf source for learned routes (#388 ) - enable changing the source address for learned ospf routes - this enables the loopback0 interface to be used as a default src address - ensure k8s nodes use loopback0 as default src - ensure incus nodes use loopback0 as default src Reviewed-on: #388	2025-09-07 16:09:21 +10:00
unkinben	ae4eb3a5eb	fix: set loopback0 as source for consul (#387 ) - fix consul service checks for prodnxsr0001-0008 - ensure the loopback0 interface whats bound too Reviewed-on: #387	2025-09-07 15:48:27 +10:00
unkinben	65fb52da55	chore: add user for jelly (#385 ) Reviewed-on: #385	2025-09-04 20:09:43 +10:00
unkinben	d97cbfd570	chore: update src ips for arr stack (#384 ) - allow the arr stack to reach prowlarr Reviewed-on: #384	2025-08-31 18:52:47 +10:00
unkinben	8f5d102945	feat: enabling changing ip for consul client (#383 ) - enable ability to set consul client bind/advertise ip Reviewed-on: #383	2025-08-14 22:55:35 +10:00
unkinben	62aade77ff	feat: add ceph-dashboard to haproxy (#382 ) - add profile to export haproxy backend - add new cert for dashboard.ceph.unkin.net - extend balancemember with ipaddress attribute Reviewed-on: #382	2025-08-14 11:06:11 +10:00
unkinben	83bb3e1085	chore: increase client body size for s3 (#381 ) - s3 clients send objects too large for the default body size Reviewed-on: #381	2025-08-13 16:41:39 +10:00
unkinben	92728047e7	feat: add ceph rgw (#380 ) - start managing ceph configuration file - manage ceph-radosgw - merge the ceph::conf and ceph::node profiles - ensure the ceph repos exist - mange nginx frontend and consul service Reviewed-on: #380	2025-08-13 12:33:41 +10:00
unkinben	f4af5e7b64	chore: add rados gateway role (#379 ) - just enough role to deploy some containers Reviewed-on: #379	2025-08-10 19:31:39 +10:00
unkinben	308d97d783	feat: enable plugins for grafana (#378 ) - add method to install plugins for grafana - ensure victoriametrics-logs-datasource is installed Reviewed-on: #378	2025-08-09 17:57:49 +10:00
				`@@ -0,0 +1 @@`
				rke2::node_token: ENC[PKCS7,MIIBuQYJKoZIhvcNAQcDoIIBqjCCAaYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWU56ILL1Te14z0vG9Q9tD9DOlRfxasZHuChZlWHiw3MLOv3FwEc9+jEW20zMJCw6xoTITJ6mXN1fXe48fT3hX5jaBijoqLpfWp64zXomXCEyLh+DZn1N8ETRFS/gB6Fk1PjkCv4x4jd92tTI28gQQAJCb5oNTJzeCFJtovECSL2ElpRJKvdkf+fFtTHdpaTcX4oASz2WUyVux8oAwkGUCRlulzpAJhcwCYoKVAq1wiFRaicK/vZ9DJBlN1HrixKNCsKwRYLZYg0Qh0NjjJPI3UlrR6l06qnTvHhYAOg2BvfQIVlcUI5NjOfUD96GqTb8hFoWhsfhQ0GQT385QmsMlDB8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCj5SiNg/nNkDbK8gR+45sMgFCzjCs/vVqJ+dFQA7r7nVhejb/bKiRtWmEnnvDSN0zOVHAueCeImNBfWB3+ZDSPbXvjN6audmdmeMV+9rvq05Nxcsb5Lcawh6DUzkJWazGH3A==]