Convert RKE2 registries to template, disable default endpoints (#474)

## Summary
- Replace static `registries.yaml` with EPP template driven by `rke2::registries` hash
- Add `disable-default-registry-endpoint: true` to all mirrors — RKE2 will only use artifactapi and never fall back to upstream registries
- Registry configuration now fully managed via hiera data (`roles/infra/k8s.yaml`)

Reviewed-on: #474
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>

.gitea/workflows/build.yaml

@@ -1,24 +0,0 @@
-name: Build
-
-on:
-  pull_request:
-
-jobs:
-  precommit:
-    runs-on: almalinux-8
-    container:
-      image: git.unkin.net/unkin/almalinux9-actionsdind:latest
-      options: --privileged
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Install requirements
-        run: |
-          dnf groupinstall -y "Development Tools" -y
-          dnf install rubygems ruby-devel gcc make redhat-rpm-config glibc-headers glibc-devel -y
-
-      - name: Pre-Commit All Files
-        run: |
-          uvx pre-commit run --all-files

.woodpecker/bolt-validate.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: bolt-validate
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/bolt-validate.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/epp-validate.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: epp-validate
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/epp-validate.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/erb-validate.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: erb-validate
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/erb-validate.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/puppet-lint.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: puppet-lint
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/puppet-lint.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/puppet-validate.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: puppet-validate
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/puppet-validate.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 2
+          limits:
+            memory: 2Gi
+            cpu: 2

.woodpecker/ruby-check.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: ruby-check
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/ruby-check.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/ruby-validate.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: ruby-validate
+    image: git.unkin.net/unkin/almalinux9-puppet-pr-validator:20260317
+    commands:
+      - uvx pre-commit run --all-files --config ci/ruby-validate.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

.woodpecker/yamllint.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: yamllint
+    image: git.unkin.net/unkin/almalinux9-base:20260606
+    commands:
+      - uvx pre-commit run --all-files --config ci/yamllint.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 1

Puppetfile

@@ -53,7 +53,6 @@ mod 'saz-ssh', '13.1.0'
 mod 'saz-limits', '5.0.0'
 mod 'ghoneycutt-timezone', '4.0.0'
 mod 'ghoneycutt-puppet', '3.3.0'
-mod 'dalen-puppetdbquery', '3.0.1'
 mod 'markt-galera', '3.1.0'
 mod 'kogitoapp-minio', '1.1.4'
 mod 'broadinstitute-certs', '3.0.1'
@@ -66,5 +65,5 @@ mod 'thias-sysctl', '1.0.8'
 mod 'cirrax-dovecot', '1.3.3'
 
 mod 'bind',
-  :git => 'https://git.service.au-syd1.consul/unkinben/puppet-bind.git',
+  :git => 'https://git.unkin.net/unkinben/puppet-bind.git',
   :tag => '1.0'

ci/bolt-validate.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: bolt-validate

ci/epp-validate.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: epp-validate

ci/erb-validate.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: erb-validate

ci/puppet-lint.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: puppet-lint
+        args:
+          - --no-80chars-check
+          - --no-documentation-check
+          - --no-puppet_url_without_modules-check
+          - --fail-on-warnings

ci/puppet-validate.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: puppet-validate

ci/ruby-check.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: ruby-validate
+  - repo: 'https://github.com/jumanjihouse/pre-commit-hooks'
+    rev: 3.0.0
+    hooks:
+      - id: reek
+      - id: rubocop

ci/ruby-validate.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: 'https://github.com/chriskuehl/puppet-pre-commit-hooks.git'
+    rev: v2.2.0
+    hooks:
+      - id: ruby-validate

ci/yamllint.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: 'https://github.com/adrienverge/yamllint'
+    rev: v1.32.0
+    hooks:
+      - id: 'yamllint'
+        args:
+          [
+            "-d {extends: relaxed, rules: {line-length: disable}, ignore: chart}",
+            "-s",
+          ]

doc/ceph/README.md

@@ -28,6 +28,98 @@ Always refer back to the official documentation at https://docs.ceph.com/en/late
     sudo ceph fs set mediafs max_mds 2
 ```
 
+
+## managing cephfs with subvolumes
+
+Create erasure code profiles. The K and M values are equivalent to the number of data disks (K) and parity disks (M) in RAID5, RAID6, etc.
+
+    sudo ceph osd erasure-code-profile set ec_6_2 k=6 m=2
+    sudo ceph osd erasure-code-profile set ec_4_1 k=4 m=1
+
+Create data pools using the erasure-code-profile, set some required options
+
+    sudo ceph osd pool create cephfs_data_ssd_ec_6_2 erasure ec_6_2
+    sudo ceph osd pool set cephfs_data_ssd_ec_6_2 allow_ec_overwrites true
+    sudo ceph osd pool set cephfs_data_ssd_ec_6_2 bulk true
+
+    sudo ceph osd pool create cephfs_data_ssd_ec_4_1 erasure ec_4_1
+    sudo ceph osd pool set cephfs_data_ssd_ec_4_1 allow_ec_overwrites true
+    sudo ceph osd pool set cephfs_data_ssd_ec_4_1 bulk true
+
+Add the pool to the fs `cephfs`
+
+    sudo ceph fs add_data_pool cephfs cephfs_data_ssd_ec_6_2
+    sudo ceph fs add_data_pool cephfs cephfs_data_ssd_ec_4_1
+
+Create a subvolumegroup using the new data pool
+
+    sudo ceph fs subvolumegroup create cephfs csi_ssd_ec_6_2 --pool_layout cephfs_data_ssd_ec_6_2
+    sudo ceph fs subvolumegroup create cephfs csi_ssd_ec_4_1 --pool_layout cephfs_data_ssd_ec_4_1
+
+All together:
+
+    sudo ceph osd erasure-code-profile set ec_6_2 k=6 m=2
+    sudo ceph osd pool create cephfs_data_ssd_ec_6_2 erasure ec_6_2
+    sudo ceph osd pool set cephfs_data_ssd_ec_6_2 allow_ec_overwrites true
+    sudo ceph osd pool set cephfs_data_ssd_ec_6_2 bulk true
+    sudo ceph fs add_data_pool cephfs cephfs_data_ssd_ec_6_2
+    sudo ceph fs subvolumegroup create cephfs csi_ssd_ec_6_2 --pool_layout cephfs_data_ssd_ec_6_2
+
+    sudo ceph osd erasure-code-profile set ec_4_1 k=4 m=1
+    sudo ceph osd pool create cephfs_data_ssd_ec_4_1 erasure ec_4_1
+    sudo ceph osd pool set cephfs_data_ssd_ec_4_1 allow_ec_overwrites true
+    sudo ceph osd pool set cephfs_data_ssd_ec_4_1 bulk true
+    sudo ceph fs add_data_pool cephfs cephfs_data_ssd_ec_4_1
+    sudo ceph fs subvolumegroup create cephfs csi_ssd_ec_4_1 --pool_layout cephfs_data_ssd_ec_4_1
+
+Create a key with access to the new subvolume groups. Check if the user already exists first:
+
+    sudo ceph auth get client.kubernetes-cephfs
+
+If it doesnt:
+
+    sudo ceph auth get-or-create client.kubernetes-cephfs \
+      mgr 'allow rw' \
+      osd 'allow rw tag cephfs metadata=cephfs, allow rw tag cephfs data=cephfs' \
+      mds 'allow r fsname=cephfs path=/volumes, allow rws fsname=cephfs path=/volumes/csi_ssd_ec_6_2, allow rws fsname=cephfs path=/volumes/csi_ssd_ec_4_1' \
+      mon 'allow r fsname=cephfs'
+
+If it does, use `sudo ceph auth caps client.kubernetes-cephfs ...` instead to update existing capabilities.
+
+## removing a cephfs subvolumegroup from cephfs
+
+This will cleanup the subvolumegroup, and subvolumes if they exist, then remove the pool.
+
+Check for subvolumegroups first, then for subvolumes in it
+
+    sudo ceph fs subvolumegroup ls cephfs
+    sudo ceph fs subvolume ls cephfs --group_name csi_raid6
+
+
+If subvolumes exist, remove each one-by-one:
+
+    sudo ceph fs subvolume rm cephfs <subvol_name> --group_name csi_raid6
+
+If you have snapshots, remove snapshots first:
+
+    sudo ceph fs subvolume snapshot ls cephfs <subvol_name> --group_name csi_raid6
+    sudo ceph fs subvolume snapshot rm cephfs <subvol_name> <snap_name> --group_name csi_raid6
+
+Once the group is empty, remove it:
+
+    sudo ceph fs subvolumegroup rm cephfs csi_raid6
+
+If it complains it’s not empty, go back as there’s still a subvolume or snapshot.
+
+If you added it with `ceph fs add_data_pool`. Undo with `rm_data_pool`:
+
+    sudo ceph fs rm_data_pool cephfs cephfs_data_csi_raid6
+
+After it’s detached from CephFS, you can delete it.
+
+    sudo ceph osd pool rm cephfs_data_csi_raid6 cephfs_data_csi_raid6 --yes-i-really-really-mean-it
+
+
 ## creating authentication tokens
 
 - this will create a client keyring named media

hiera.yaml

@@ -30,6 +30,7 @@ hierarchy:
       - "roles/%{::enc_role_tier1}.eyaml"
       - "roles/%{::enc_role_tier1}.yaml"
       - "virtual/%{facts.virtual}.yaml"
+      - "os/%{facts.os.name}/%{facts.os.name}%{facts.os.release.major}.%{facts.os.release.minor}.yaml"
       - "os/%{facts.os.name}/%{facts.os.name}%{facts.os.release.major}.yaml"
       - "os/%{facts.os.name}/all_releases.yaml"
       - "common.eyaml"

hieradata/country/au/region/syd1/infra/halb/haproxy2.yaml

@@ -1,4 +1,7 @@
 ---
+haproxy_server_k8s_syd1_traefik_internal: 'k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2'
+haproxy_server_k8s_syd1_traefik_external: 'k8s-traefik-external 198.18.199.0:443 ssl verify none check inter 2s rise 3 fall 2'
+
 profiles::haproxy::dns::ipaddr: "%{hiera('anycast_ip')}"
 profiles::haproxy::dns::vrrp_cnames:
   - sonarr.main.unkin.net
@@ -16,6 +19,7 @@ profiles::haproxy::dns::vrrp_cnames:
   - mail.main.unkin.net
   - autoconfig.main.unkin.net
   - autodiscover.main.unkin.net
+  - auth.unkin.net
 
 profiles::haproxy::mappings:
   fe_http:
@@ -37,6 +41,7 @@ profiles::haproxy::mappings:
       - 'mail-webadmin.main.unkin.net be_stalwart_webadmin'
       - 'autoconfig.main.unkin.net be_stalwart_webadmin'
       - 'autodiscovery.main.unkin.net be_stalwart_webadmin'
+      - 'auth.unkin.net be_k8s_kanidm'
   fe_https:
     ensure: present
     mappings:
@@ -56,6 +61,7 @@ profiles::haproxy::mappings:
       - 'mail-webadmin.main.unkin.net be_stalwart_webadmin'
       - 'autoconfig.main.unkin.net be_stalwart_webadmin'
       - 'autodiscovery.main.unkin.net be_stalwart_webadmin'
+      - 'auth.unkin.net be_k8s_kanidm'
 
 profiles::haproxy::frontends:
   fe_http:
@@ -80,6 +86,7 @@ profiles::haproxy::frontends:
         - 'acl_stalwart_webadmin req.hdr(host) -i mail-webadmin.main.unkin.net'
         - 'acl_stalwart_webadmin req.hdr(host) -i autoconfig.main.unkin.net'
         - 'acl_stalwart_webadmin req.hdr(host) -i autodiscovery.main.unkin.net'
+        - 'acl_kanidm req.hdr(host) -i auth.unkin.net'
         - 'acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24'
       use_backend:
         - "%[req.hdr(host),lower,map(/etc/haproxy/fe_https.map,be_default)]"
@@ -99,6 +106,7 @@ profiles::haproxy::frontends:
         - 'set-header X-Frame-Options DENY if acl_grafana'
         - 'set-header X-Frame-Options DENY if acl_ceph_dashboard'
         - 'set-header X-Frame-Options DENY if acl_stalwart_webadmin'
+        - 'set-header X-Frame-Options DENY if acl_kanidm'
         - 'set-header X-Content-Type-Options nosniff'
         - 'set-header X-XSS-Protection 1;mode=block'
 
@@ -320,6 +328,26 @@ profiles::haproxy::backends:
         - add-header X-Forwarded-Proto https if { dst_port 9443 }
       redirect: 'scheme https if !{ ssl_fc }'
       stick-table: 'type ip size 200k expire 30m'
+  be_k8s_kanidm:
+    description: Backend for Kanidm (auth.unkin.net via Kubernetes internal Traefik)
+    collect_exported: false
+    options:
+      balance: roundrobin
+      option:
+        - httpchk
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      http-check:
+        - 'connect ssl sni auth.unkin.net'
+        - 'send meth GET uri /status ver HTTP/1.1 hdr Host auth.unkin.net'
+        - 'expect status 200'
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+      server: "%{lookup('haproxy_server_k8s_syd1_traefik_internal')} sni str(auth.unkin.net)"
   be_stalwart_imap:
     description: Backend for Stalwart IMAP (STARTTLS)
     collect_exported: false
@@ -393,6 +421,7 @@ profiles::haproxy::certlist::certificates:
   - /etc/pki/tls/letsencrypt/git.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/letsencrypt/grafana.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/letsencrypt/dashboard.ceph.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/auth.unkin.net/fullchain_combined.pem
   - /etc/pki/tls/vault/certificate.pem
 
 # additional altnames
@@ -422,3 +451,4 @@ certbot::client::domains:
   - git.unkin.net
   - grafana.unkin.net
   - dashboard.ceph.unkin.net
+  - auth.unkin.net

hieradata/os/AlmaLinux/AlmaLinux8.yaml

@@ -1,7 +1,7 @@
 # hieradata/os/AlmaLinux/AlmaLinux8.yaml
 ---
 crypto_policies::policy: 'DEFAULT'
-
+almalinux-base-repo: almalinux
 profiles::packages::include:
   network-scripts: {}
 

hieradata/os/AlmaLinux/AlmaLinux9.7.yaml

@@ -0,0 +1,2 @@
+---
+almalinux-base-repo: almalinux-vault

hieradata/os/AlmaLinux/AlmaLinux9.yaml

@@ -1,7 +1,7 @@
 # hieradata/os/AlmaLinux/AlmaLinux9.yaml
 ---
 crypto_policies::policy: 'DEFAULT:SHA1'
-
+almalinux-base-repo: almalinux
 profiles::yum::global::repos:
   crb:
     ensure: present

hieradata/os/AlmaLinux/all_releases.yaml

@@ -23,45 +23,45 @@ profiles::yum::global::repos:
     name: baseos
     descr: baseos repository
     target: /etc/yum.repos.d/baseos.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   extras:
     name: extras
     descr: extras repository
     target: /etc/yum.repos.d/extras.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/extras/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/extras/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/extras/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/extras/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   appstream:
     name: appstream
     descr: appstream repository
     target: /etc/yum.repos.d/appstream.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   highavailability:
     name: highavailability
     descr: highavailability repository
     target: /etc/yum.repos.d/highavailability.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   crb:
     ensure: absent
     name: crb
     descr: crb repository
     target: /etc/yum.repos.d/crb.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   powertools:
     ensure: absent
     name: powertools
     descr: powertools repository
     target: /etc/yum.repos.d/powertools.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os/
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/%{lookup('almalinux-base-repo')}/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   epel:
     name: epel

hieradata/roles/infra/auth/glauth.yaml

@@ -66,6 +66,9 @@ glauth::users:
       - 20025   # jupyterhub_admin
       - 20026   # jupyterhub_user
       - 20027   # grafana_user
+      - 20028   # k8s/au/syd1 operator
+      - 20029   # k8s/au/syd1 admin
+      - 20030   # k8s/au/syd1 root
     loginshell: '/bin/bash'
     homedir: '/home/benvin'
     passsha256: 'd2434f6b4764ef75d5b7b96a876a32deedbd6aa726a109c3f32e823ca66f604a'
@@ -385,3 +388,12 @@ glauth::groups:
   grafana_user:
     group_name: 'grafana_user'
     gidnumber: 20027
+  kubernetes_au_syd1_cluster_operator:
+    group_name: 'kubernetes_au_syd1_cluster_operator'
+    gidnumber: 20028
+  kubernetes_au_syd1_cluster_admin:
+    group_name: 'kubernetes_au_syd1_cluster_admin'
+    gidnumber: 20029
+  kubernetes_au_syd1_cluster_root:
+    group_name: 'kubernetes_au_syd1_cluster_root'
+    gidnumber: 20030

hieradata/roles/infra/dns/resolver.yaml

@@ -6,8 +6,10 @@ hiera_include:
 profiles::dns::resolver::acls:
   acl-main.unkin.net:
     addresses:
-      - 10.10.8.1/32
+      - 198.18.1.10/32
+      - 198.18.2.160/27
       - 198.18.21.160/27
+      - 198.18.2.192/27
       - 198.18.21.192/27
       - 198.18.13.0/24
       - 198.18.14.0/24

hieradata/roles/infra/git/server.yaml

@@ -82,8 +82,14 @@ profiles::sql::postgresdb::dbname: gitea
 profiles::sql::postgresdb::dbuser: gitea
 
 # deploy gitea
-gitea::ensure: '1.22.4'
-gitea::checksum: 'd549104f55067e6fb156e7ba060c9af488f36e12d5e747db7563fcc99eaf8532'
+gitea::base_url: 'https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/gitea-dl/gitea'
+gitea::install::checksums:
+  1.26.2:
+    linux:
+      amd64: 5b37937b625de69196748f7293eee1a7363f8637ae6e3da3c359bb380bd61a6a
+
+gitea::ensure: '1.26.2'
+gitea::checksum: '5b37937b625de69196748f7293eee1a7363f8637ae6e3da3c359bb380bd61a6a'
 gitea::manage_user: false
 gitea::manage_group: false
 gitea::manage_home: false

hieradata/roles/infra/incus/node.yaml

@@ -5,6 +5,10 @@ hiera_include:
   - incus
   - zfs
   - profiles::ceph::node
+  - profiles::ceph::mon
+  - profiles::ceph::mgr
+  - profiles::ceph::mds
+  - profiles::ceph::osd
   - profiles::ceph::client
   - profiles::ceph::dashboard
   - profiles::storage::cephfsvols
@@ -99,7 +103,7 @@ profiles::yum::global::repos:
 profiles::dns::base::primary_interface: loopback0
 
 # dashboard/haproxy
-profiles::ceph::dashboard::ipaddress: "%{hiera('networking_loopback0_ip')}"
+profiles::ceph::dashboard::ipaddress: "%{hiera('networking_loopback2_ip')}"
 
 # networking
 systemd::manage_networkd: true

hieradata/roles/infra/k8s.yaml

@@ -2,6 +2,7 @@
 hiera_include:
   - profiles::selinux::setenforce
   - profiles::ceph::node
+  - profiles::ceph::osd
   - profiles::ceph::client
   - exporters::frr_exporter
   - frrouting
@@ -10,6 +11,62 @@ hiera_include:
 # manage rke2
 rke2::bootstrap_node: prodnxsr0001.main.unkin.net
 rke2::join_url: https://join-k8s.service.consul:9345
+rke2::manage_registries: true
+rke2::registries:
+  docker.io:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "dockerhub/$1"
+    disable-default-registry-endpoint: true
+  ghcr.io:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "ghcr/$1"
+    disable-default-registry-endpoint: true
+  quay.io:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "quay/$1"
+    disable-default-registry-endpoint: true
+  registry.k8s.io:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "k8s-registry/$1"
+    disable-default-registry-endpoint: true
+  registry.gitlab.com:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "gitlab/$1"
+    disable-default-registry-endpoint: true
+  docker.elastic.co:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "elastic/$1"
+    disable-default-registry-endpoint: true
+  gcr.io:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "gcr/$1"
+    disable-default-registry-endpoint: true
+  docker.litellm.ai:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "litellm/$1"
+    disable-default-registry-endpoint: true
+  public.ecr.aws:
+    endpoint:
+      - "https://artifactapi.k8s.syd1.au.unkin.net"
+    rewrite:
+      "^(.*)$": "ecr-public/$1"
+    disable-default-registry-endpoint: true
 rke2::config_hash:
   bind-address: "%{hiera('networking_loopback0_ip')}"
   node-ip: "%{hiera('networking_loopback0_ip')}"
@@ -125,6 +182,17 @@ frrouting::ospf_exclude_k8s_enable: true
 frrouting::k8s_cluster_cidr: '10.42.0.0/16'  # RKE2 cluster-cidr (pods)
 frrouting::k8s_service_cidr: '10.43.0.0/16'  # RKE2 service-cidr
 
+# sysctl recommendations
+sysctl::base::values:
+  net.ipv4.conf.default.rp_filter:
+    value: '0'
+  net.ipv4.conf.all.rp_filter:
+    value: '0'
+  fs.inotify.max_user_watches:
+    value: '524288'
+  fs.inotify.max_user_instances:
+    value: '512'
+
 # add loopback interfaces to ssh list
 ssh::server::options:
   ListenAddress:

hieradata/roles/infra/metrics/grafana.yaml

@@ -11,6 +11,7 @@ profiles::metrics::grafana::db_name: "%{hiera('profiles::sql::postgresdb::dbname
 profiles::metrics::grafana::db_user: "%{hiera('profiles::sql::postgresdb::dbuser')}"
 profiles::metrics::grafana::db_pass: "%{hiera('profiles::sql::postgresdb::dbpass')}"
 profiles::metrics::grafana::pgsql_backend: true
+profiles::metrics::grafana::version: '13.0.2'
 profiles::metrics::grafana::plugins:
   victoriametrics-logs-datasource:
     ensure: present

hieradata/roles/infra/pki/certbot.yaml

@@ -16,3 +16,4 @@ certbot::domains:
   - git.unkin.net
   - grafana.unkin.net
   - dashboard.ceph.unkin.net
+  - auth.unkin.net

hieradata/roles/infra/puppet/master.yaml

@@ -26,7 +26,7 @@ profiles::puppet::cobbler_enc::packages:
   - 'requests'
   - 'PyYAML'
 profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-enc.git
-profiles::puppet::r10k::r10k_repo: https://git.service.au-syd1.consul/unkin/puppet-r10k.git
+profiles::puppet::r10k::r10k_repo: https://git.unkin.net/unkin/puppet-r10k.git
 profiles::puppet::g10k::bin_path: '/usr/bin/g10k'
 profiles::puppet::g10k::cfg_path: '/etc/puppetlabs/r10k/r10k.yaml'
 profiles::puppet::g10k::environments_path: '/etc/puppetlabs/code/environments'

hieradata/roles/infra/storage/consul.yaml

@@ -29,6 +29,7 @@ profiles::consul::server::acl:
 profiles::pki::vault::alt_names:
   - consul.main.unkin.net
   - consul.service.consul
+  - "consul.service.%{facts.country}-%{facts.region}.consul"
   - consul
 
 # manage a simple nginx reverse proxy

hieradata/roles/infra/storage/vault.yaml

@@ -2,10 +2,12 @@
 profiles::vault::server::members_role: roles::infra::storage::vault
 profiles::vault::server::members_lookup: true
 profiles::vault::server::data_dir: /data/vault
+profiles::vault::server::plugin_dir: /opt/openbao-plugins
 profiles::vault::server::manage_storage_dir: true
 profiles::vault::server::tls_disable: false
-vault::package_name: openbao
-vault::package_ensure: latest
+profiles::vault::server::package_name: openbao
+profiles::vault::server::package_ensure: 2.4.4
+profiles::vault::server::disable_openbao: false
 
 # additional altnames
 profiles::pki::vault::alt_names:
@@ -23,3 +25,6 @@ profiles::nginx::simpleproxy::proxy_scheme: 'http'
 profiles::nginx::simpleproxy::proxy_host: '127.0.0.1'
 profiles::nginx::simpleproxy::proxy_port: 8200
 profiles::nginx::simpleproxy::proxy_path: '/'
+
+profiles::packages::include:
+  openbao-plugins: {}

modules/externaldns/manifests/master.pp

@@ -28,8 +28,8 @@ class externaldns::master inherits externaldns {
       dynamic         => true,
       allow_updates   => ['key externaldns-key'],
       allow_transfers => empty($slave_ips) ? {
-        true  => [],
-        false => ['dns-slaves'],
+        true  => ['key externaldns-key'],
+        false => ['key externaldns-key','dns-slaves'],
       },
       ns_notify       => !empty($slave_ips),
       also_notify     => $slave_ips,

modules/incus/manifests/cluster.pp

@@ -22,7 +22,12 @@ class incus::cluster (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

modules/libs/lib/facter/ceph_services.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'facter'
+
+# Detects active ceph service instances via systemctl and exposes facts
+# for use in ceph service management profiles.
+# rubocop:disable Style/ClassAndModuleChildren
+module Unkin
+  module Ceph
+    # Detects active ceph service instances via systemctl and exposes Facter facts.
+    module Utils
+      TYPES = %w[mon mgr mds osd].freeze
+
+      def self.services
+        output = Facter::Core::Execution.execute(
+          'systemctl list-units "ceph*" --no-legend --plain --all 2>/dev/null',
+          on_fail: ''
+        )
+        parse_units(output)
+      end
+
+      def self.parse_units(output)
+        result = TYPES.each_with_object({}) { |type, hash| hash[type] = [] }
+        output.each_line do |line|
+          unit = line.split.first
+          next unless unit
+
+          match_unit(result, unit)
+        end
+        result
+      end
+
+      def self.match_unit(result, unit)
+        TYPES.each do |type|
+          match = unit.match(/\Aceph-#{type}@(.+)\.service\z/)
+          result[type] << "ceph-#{type}@#{match[1]}" if match
+        end
+      end
+
+      TYPES.each do |type|
+        define_singleton_method(:"#{type}?") { !services[type].empty? }
+      end
+    end
+  end
+end
+# rubocop:enable Style/ClassAndModuleChildren
+
+Facter.add('ceph_services') do
+  setcode { Unkin::Ceph::Utils.services }
+end
+
+Unkin::Ceph::Utils::TYPES.each do |type|
+  Facter.add("is_ceph_#{type}") do
+    setcode { Unkin::Ceph::Utils.public_send(:"#{type}?") }
+  end
+end

modules/redisha/manifests/redis.pp

@@ -20,7 +20,12 @@ class redisha::redis (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${redisha_members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

modules/redisha/manifests/sentinel.pp

@@ -23,7 +23,12 @@ class redisha::sentinel (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${redisha_members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

modules/rke2/manifests/config.pp

@@ -7,6 +7,8 @@ class rke2::config (
   Stdlib::Fqdn            $bootstrap_node = $rke2::bootstrap_node,
   String                  $node_token = $rke2::node_token,
   Array[String[1]]        $extra_config_files = $rke2::extra_config_files,
+  Boolean                 $manage_registries = $rke2::manage_registries,
+  Hash                    $registries = $rke2::registries,
 ){
 
   # if its not the bootstrap node, add join path to config
@@ -28,6 +30,24 @@ class rke2::config (
     $config = $config_hash
   }
 
+  if $manage_registries {
+    file { '/etc/rancher/rke2/registries.yaml':
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => epp('rke2/registries.yaml.epp', { registries => $registries }),
+      require => Package["rke2-${node_type}"],
+      notify  => Service["rke2-${node_type}"],
+    }
+  }else{
+    file { '/etc/rancher/rke2/registries.yaml':
+      ensure  => absent,
+      require => Package["rke2-${node_type}"],
+      notify  => Service["rke2-${node_type}"],
+    }
+  }
+
   # create the config file
   file { $config_file:
     ensure  => file,

modules/rke2/manifests/init.pp

@@ -12,6 +12,8 @@ class rke2 (
   Hash                    $helm_repos = $rke2::params::helm_repos,
   Array[String[1]]        $extra_config_files = $rke2::params::extra_config_files,
   Stdlib::HTTPUrl         $container_archive_source = $rke2::params::container_archive_source,
+  Boolean                 $manage_registries = $rke2::params::manage_registries,
+  Hash                    $registries = $rke2::params::registries,
 ) inherits rke2::params {
 
   include rke2::install

modules/rke2/manifests/params.pp

@@ -12,4 +12,6 @@ class rke2::params (
   Hash                    $helm_repos = {},
   Array[String[1]]        $extra_config_files = [],
   Stdlib::HTTPUrl         $container_archive_source = 'https://github.com/rancher/rke2/releases/download',
+  Boolean                 $manage_registries = false,
+  Hash                    $registries = {},
 ) {}

modules/rke2/templates/registries.yaml.epp

@@ -0,0 +1,20 @@
+<%- | Hash $registries | -%>
+---
+# DO NOT MODIFY - MANAGED BY PUPPET
+mirrors:
+<%- $registries.each |$registry, $config| { -%>
+  <%= $registry %>:
+    endpoint:
+<%- $config['endpoint'].each |$ep| { -%>
+      - "<%= $ep %>"
+<%- } -%>
+<%- if $config['rewrite'] { -%>
+    rewrite:
+<%- $config['rewrite'].each |$pattern, $replacement| { -%>
+      "<%= $pattern %>": "<%= $replacement %>"
+<%- } -%>
+<%- } -%>
+<%- if $config['disable-default-registry-endpoint'] { -%>
+    disable-default-registry-endpoint: true
+<%- } -%>
+<%- } -%>

modules/stalwart/manifests/init.pp

@@ -167,7 +167,13 @@ class stalwart (
 
   # Query cluster members for validation
   $cluster_query = "enc_role='${cluster_role}' and country='${facts['country']}' and region='${facts['region']}'"
-  $cluster_members_raw = query_nodes($cluster_query, 'networking.fqdn')
+  $cluster_members_raw = puppetdb_query(
+    "facts[certname] {
+      name = 'enc_role' and value = '${cluster_role}' and
+      certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and
+      certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+    }"
+  ).map |$fact| { $fact['certname'] }
   $cluster_members = $cluster_members_raw ? {
     undef   => [],
     default => $cluster_members_raw,
@@ -180,7 +186,20 @@ class stalwart (
 
   # Query HAProxy nodes for proxy trusted networks
   $haproxy_query = "enc_role='${haproxy_role}' and country='${facts['country']}' and region='${facts['region']}'"
-  $haproxy_members_raw = query_nodes($haproxy_query, 'networking.ip')
+  $haproxy_members_raw = puppetdb_query(
+    "facts[certname,value] {
+      name = 'networking' and
+      certname in facts[certname] {
+        name = 'enc_role' and value = '${haproxy_role}'
+      } and
+      certname in facts[certname] {
+        name = 'country' and value = '${facts['country']}'
+      } and
+      certname in facts[certname] {
+        name = 'region' and value = '${facts['region']}'
+      }
+    }"
+  ).map |$fact| { $fact['value']['ip'] }
   $haproxy_ips = $haproxy_members_raw ? {
     undef   => [],
     default => sort($haproxy_members_raw),

site/profiles/manifests/ceph/mds.pp

@@ -0,0 +1,13 @@
+class profiles::ceph::mds (
+  Boolean $ensure_running = true,
+) {
+
+  if $ensure_running and $facts['is_ceph_mds'] {
+    $facts['ceph_services']['mds'].each |String $svc| {
+      service { $svc:
+        ensure => running,
+        enable => true,
+      }
+    }
+  }
+}

site/profiles/manifests/ceph/mgr.pp

@@ -0,0 +1,13 @@
+class profiles::ceph::mgr (
+  Boolean $ensure_running = true,
+) {
+
+  if $ensure_running and $facts['is_ceph_mgr'] {
+    $facts['ceph_services']['mgr'].each |String $svc| {
+      service { $svc:
+        ensure => running,
+        enable => true,
+      }
+    }
+  }
+}

site/profiles/manifests/ceph/mon.pp

@@ -0,0 +1,13 @@
+class profiles::ceph::mon (
+  Boolean $ensure_running = true,
+) {
+
+  if $ensure_running and $facts['is_ceph_mon'] {
+    $facts['ceph_services']['mon'].each |String $svc| {
+      service { $svc:
+        ensure => running,
+        enable => true,
+      }
+    }
+  }
+}

site/profiles/manifests/ceph/osd.pp

@@ -0,0 +1,13 @@
+class profiles::ceph::osd (
+  Boolean $ensure_running = true,
+) {
+
+  if $ensure_running and $facts['is_ceph_osd'] {
+    $facts['ceph_services']['osd'].each |String $svc| {
+      service { $svc:
+        ensure => running,
+        enable => true,
+      }
+    }
+  }
+}

site/profiles/manifests/consul/client.pp

@@ -28,7 +28,12 @@ class profiles::consul::client (
       }
 
       # if it is, find hosts, sort them so they dont cause changes every run
-      $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn'))
+      $servers_array = sort(puppetdb_query(
+        "facts[certname] {
+          name = 'enc_role' and value = '${members_role}' and
+          certname in facts[certname] { name = 'region' and value = '${::facts['region']}' }
+        }"
+      ).map |$fact| { $fact['certname'] })
 
     # else use provided array from params
     }else{

site/profiles/manifests/consul/server.pp

@@ -65,12 +65,22 @@ class profiles::consul::server (
       }
 
       # if it is, find hosts, sort them so they dont cause changes every run
-      $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn'))
+      $servers_array = sort(puppetdb_query(
+        "facts[certname] {
+          name = 'enc_role' and value = '${members_role}' and
+          certname in facts[certname] { name = 'region' and value = '${::facts['region']}' }
+        }"
+      ).map |$fact| { $fact['certname'] })
 
       if $join_remote_regions {
         # get all nodes in the members_role for each other region
         $region_to_servers = $remote_regions.reduce({}) |$memo, $region| {
-          $servers = sort(query_nodes("enc_role='${members_role}' and region='${region}'", 'networking.fqdn'))
+          $servers = sort(puppetdb_query(
+            "facts[certname] {
+              name = 'enc_role' and value = '${members_role}' and
+              certname in facts[certname] { name = 'region' and value = '${region}' }
+            }"
+          ).map |$fact| { $fact['certname'] })
           $memo + { $region => $servers }
         }
 

site/profiles/manifests/dns/base.pp

@@ -18,9 +18,28 @@ class profiles::dns::base (
   $nameserver_array = $ns_role ? {
     undef   => $nameservers,
     default => $use_ns ? {
-      'all'     => query_nodes("enc_role='${ns_role}'", 'networking.ip'),
-      'region'  => query_nodes("enc_role='${ns_role}' and region=${facts['region']}", 'networking.ip'),
-      'country' => query_nodes("enc_role='${ns_role}' and country=${facts['country']}", 'networking.ip'),
+      'all'     => puppetdb_query(
+        "facts[certname,value] {
+          name = 'networking' and
+          certname in nodes[certname] { facts.enc_role = '${ns_role}' }
+        }"
+      ).map |$fact| { $fact['value']['ip'] },
+      'region'  => puppetdb_query(
+        "facts[certname,value] {
+          name = 'networking' and
+          certname in nodes[certname] {
+            facts.enc_role = '${ns_role}' and facts.region = '${facts['region']}'
+          }
+        }"
+      ).map |$fact| { $fact['value']['ip'] },
+      'country' => puppetdb_query(
+        "facts[certname,value] {
+          name = 'networking' and
+          certname in nodes[certname] {
+            facts.enc_role = '${ns_role}' and facts.country = '${facts['country']}'
+          }
+        }"
+      ).map |$fact| { $fact['value']['ip'] },
     }
   }
 

site/profiles/manifests/dns/master.pp

@@ -20,9 +20,21 @@ class profiles::dns::master (
   $nameservers_array = $ns_role ? {
     undef   => [$facts['networking']['fqdn']],
     default => $use_ns ? {
-      'all'     => sort(query_nodes("enc_role='${ns_role}'", 'networking.fqdn')),
-      'region'  => sort(query_nodes("enc_role='${ns_role}' and region=${facts['region']}", 'networking.fqdn')),
-      'country' => sort(query_nodes("enc_role='${ns_role}' and country=${facts['country']}", 'networking.fqdn')),
+      'all'     => sort(puppetdb_query(
+        "facts[certname] { name = 'enc_role' and value = '${ns_role}' }"
+      ).map |$fact| { $fact['certname'] }),
+      'region'  => sort(puppetdb_query(
+        "facts[certname] {
+          name = 'enc_role' and value = '${ns_role}' and
+          certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+        }"
+      ).map |$fact| { $fact['certname'] }),
+      'country' => sort(puppetdb_query(
+        "facts[certname] {
+          name = 'enc_role' and value = '${ns_role}' and
+          certname in facts[certname] { name = 'country' and value = '${facts['country']}' }
+        }"
+      ).map |$fact| { $fact['certname'] }),
     }
   }
 
@@ -32,7 +44,9 @@ class profiles::dns::master (
       $facts['networking']['fqdn'] => $facts['networking']['ip']
     },
     default => $nameservers_array.reduce({}) |$acc, $fqdn| {
-      $result = query_nodes("networking.fqdn='${fqdn}'", 'networking.ip')
+      $result = puppetdb_query(
+        "facts[certname,value] { name = 'networking' and certname = '${fqdn}' }"
+      ).map |$fact| { $fact['value']['ip'] }
       $ip = $result[0]
       $acc + { "${fqdn}." => $ip }
     }

site/profiles/manifests/etcd/node.pp

@@ -18,7 +18,12 @@ class profiles::etcd::node (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{
@@ -31,7 +36,7 @@ class profiles::etcd::node (
     $initial_cluster = $servers_array.map |$fqdn| {
 
       # lookup the ip address for the current fqdn
-      $ip = query_nodes("networking.fqdn='${fqdn}'", 'networking.ip')[0]
+      $ip = puppetdb_query("facts[certname,value] { name = 'networking' and certname = '${fqdn}' }").map |$fact| { $fact['value']['ip'] }[0]
 
       # construct the string for this server
       "${fqdn}=https://${ip}:${peer_port}"

site/profiles/manifests/haproxy/dns.pp

@@ -30,13 +30,14 @@ class profiles::haproxy::dns (
   }
 
   # if it is, find hosts, sort them so they dont cause changes every run
-  $servers_array = sort(query_nodes(
-    "enc_role='${facts['enc_role']}' and
-      country='${facts['country']}' and
-      region='${facts['region']}' and
-      environment='${facts['environment']}'",
-    'networking.fqdn'
-  ))
+  $servers_array = sort(puppetdb_query(
+    "facts[certname] {
+      name = 'enc_role' and value = '${facts['enc_role']}' and
+      certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and
+      certname in facts[certname] { name = 'region' and value = '${facts['region']}' } and
+      certname in facts[certname] { name = 'environment' and value = '${facts['environment']}' }
+    }"
+  ).map |$fact| { $fact['certname'] })
 
   # give enough time for a few hosts to be provisioned
   if length($servers_array) >= 3 {

site/profiles/manifests/metrics/grafana.pp

@@ -1,6 +1,7 @@
 # profiles::metrics::grafana
 class profiles::metrics::grafana (
   String    $ldap_bind_pass,
+  String    $version        = 'installed',
   Stdlib::Port $http_port   = 8080,
   String    $app_mode       = 'production',
   Boolean   $allow_sign_up  = false,
@@ -107,6 +108,7 @@ class profiles::metrics::grafana (
 
   # deploy grafana
   class { 'grafana':
+    version  => $version,
     cfg      => $cfg,
     ldap_cfg => $ldap_cfg,
     plugins  => $plugins,

site/profiles/manifests/minio/server.pp

@@ -98,8 +98,15 @@ class profiles::minio::server (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    #$servers_array = sort(query_nodes("enc_role='${minio_members_role}'", 'networking.fqdn'))
-    $servers_array = sort(query_nodes("enc_role='${minio_members_role}' and minio_region='${minio_region}'", 'networking.fqdn'))
+    #$servers_array = sort(puppetdb_query(
+    #  "facts[certname] { name = 'enc_role' and value = '${minio_members_role}' }"
+    #).map |$fact| { $fact['certname'] })
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${minio_members_role}' and
+        certname in facts[certname] { name = 'minio_region' and value = '${minio_region}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

site/profiles/manifests/ntp/client.pp

@@ -26,9 +26,21 @@ class profiles::ntp::client (
     $ntpserver_array = $ntp_role ? {
       undef   => $peers,
       default => $use_ntp ? {
-        'all'     => query_nodes("enc_role='${ntp_role}'", 'networking.fqdn'),
-        'region'  => query_nodes("enc_role='${ntp_role}' and region=${facts['region']}", 'networking.fqdn'),
-        'country' => query_nodes("enc_role='${ntp_role}' and country=${facts['country']}", 'networking.fqdn'),
+        'all'     => puppetdb_query(
+          "facts[certname] { name = 'enc_role' and value = '${ntp_role}' }"
+        ).map |$fact| { $fact['certname'] },
+        'region'  => puppetdb_query(
+          "facts[certname] {
+            name = 'enc_role' and value = '${ntp_role}' and
+            certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+          }"
+        ).map |$fact| { $fact['certname'] },
+        'country' => puppetdb_query(
+          "facts[certname] {
+            name = 'enc_role' and value = '${ntp_role}' and
+            certname in facts[certname] { name = 'country' and value = '${facts['country']}' }
+          }"
+        ).map |$fact| { $fact['certname'] },
       }
     }
 

site/profiles/manifests/proxmox/clusterinit.pp

@@ -24,10 +24,13 @@ class profiles::proxmox::clusterinit {
     }
   }
 
-  $servers_array = sort(query_nodes(
-    "enc_role='${membersrole}' and country='${facts['country']}' and region='${facts['region']}'",
-    'networking.fqdn'
-  ))
+  $servers_array = sort(puppetdb_query(
+    "facts[certname] {
+      name = 'enc_role' and value = '${membersrole}' and
+      certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and
+      certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+    }"
+  ).map |$fact| { $fact['certname'] })
 
   if ! $profiles::proxmox::params::pve_clusterinit_master {
     if !empty($servers_array) {

site/profiles/manifests/proxmox/clusterjoin.pp

@@ -11,13 +11,14 @@ class profiles::proxmox::clusterjoin {
   $root_password      = $profiles::proxmox::params::root_password
 
   # query puppetdb for list of cluster members
-  $members_array = sort(query_nodes(
-    "enc_role='${membersrole}' and \
-    country='${facts['country']}' and \
-    region='${facts['region']}' and \
-    pve_cluster.cluster_name='${clustername}'",
-    'networking.fqdn'
-  ))
+  $members_array = sort(puppetdb_query(
+    "facts[certname] {
+      name = 'enc_role' and value = '${membersrole}' and
+      certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and
+      certname in facts[certname] { name = 'region' and value = '${facts['region']}' } and
+      certname in facts[certname] { name = 'pve_cluster' and value.cluster_name = '${clustername}' }
+    }"
+  ).map |$fact| { $fact['certname'] })
 
   # check if the pve kernerl is running
   if $facts['kernelrelease'] == $profiles::proxmox::params::pve_kernel_release {

site/profiles/manifests/puppet/puppetboard.pp

@@ -3,7 +3,7 @@
 # This class manages the Puppetboard, a web interface to PuppetDB.
 #
 class profiles::puppet::puppetboard (
-  String  $python_version       = $facts['python3_release'],
+  String  $python_version       = '3.12',
   Boolean $manage_virtualenv    = false,
   Integer $reports_count        = 40,
   Boolean $offline_mode         = true,

site/profiles/manifests/sql/galera_member.pp

@@ -48,7 +48,12 @@ class profiles::sql::galera_member (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${galera_members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${galera_members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

site/profiles/manifests/sql/postgresdb.pp

@@ -18,7 +18,12 @@ class profiles::sql::postgresdb (
     }
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{

site/profiles/manifests/vault/server.pp

@@ -6,11 +6,15 @@ class profiles::vault::server (
     Undef
   ]         $members_role   = undef,
   Array     $vault_servers  = [],
+  String        $package_name       = 'vault',
+  String        $package_ensure     = 'latest',
+  Boolean       $disable_openbao    = true,
   Boolean       $tls_disable        = false,
   Stdlib::Port  $client_port        = 8200,
   Stdlib::Port  $cluster_port       = 8201,
   Boolean   $manage_storage_dir     = false,
   Stdlib::Absolutepath $data_dir    = '/opt/vault',
+  Stdlib::Absolutepath $plugin_dir  = '/opt/vault_plugins',
   Stdlib::Absolutepath $bin_dir     = '/usr/bin',
   Stdlib::Absolutepath $ssl_crt     = '/etc/pki/tls/vault/certificate.crt',
   Stdlib::Absolutepath $ssl_key     = '/etc/pki/tls/vault/private.key',
@@ -25,7 +29,12 @@ class profiles::vault::server (
   if $members_lookup and $members_role != undef {
 
     # if it is, find hosts, sort them so they dont cause changes every run
-    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn'))
+    $servers_array = sort(puppetdb_query(
+      "facts[certname] {
+        name = 'enc_role' and value = '${members_role}' and
+        certname in facts[certname] { name = 'region' and value = '${::facts['region']}' }
+      }"
+    ).map |$fact| { $fact['certname'] })
 
   # else use provided array from params
   }else{
@@ -51,7 +60,33 @@ class profiles::vault::server (
       }
     }
 
+    # cleanup openbao?
+    if $disable_openbao {
+      package {'openbao':
+        ensure => absent,
+        before => Class['vault']
+      }
+      package {'openbao-vault-compat':
+        ensure => absent,
+        before => [
+          Class['vault'],
+          Package['openbao']
+        ]
+      }
+    }
+
+    # add versionlock for package_name?
+    if $package_ensure != 'latest' {
+      yum::versionlock{$package_name:
+        ensure  => present,
+        version => $package_ensure,
+        before  => Class['vault']
+      }
+    }
+
     class { 'vault':
+      package_name       => $package_name,
+      package_ensure     => $package_ensure,
       manage_service     => false,
       manage_storage_dir => $manage_storage_dir,
       enable_ui          => true,
@@ -65,6 +100,7 @@ class profiles::vault::server (
       api_addr           => "${http_scheme}://${::facts['networking']['fqdn']}:${client_port}",
       extra_config       => {
         cluster_addr     => "${http_scheme}://${::facts['networking']['fqdn']}:${cluster_port}",
+        plugin_directory => $plugin_dir,
       },
       listener           => [
         {