From ae256b7c0bc7072cadf6896521d93eb330961aa3 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 21 Mar 2026 21:54:24 +1100 Subject: [PATCH] fix: replace puppetdbquery with native PQL queries Replace deprecated dalen-puppetdbquery module with native puppetdb_query function using PQL syntax to resolve URI.escape compatibility issues. This is required to migrated to Puppet 8 (and kubernetes). Changes: - Remove dalen-puppetdbquery dependency from Puppetfile - Replace query_nodes() calls with puppetdb_query() using PQL syntax - Update 27 function calls across 18 Puppet manifests - Maintain equivalent functionality with improved compatibility --- Puppetfile | 1 - modules/incus/manifests/cluster.pp | 7 +++++- modules/redisha/manifests/redis.pp | 7 +++++- modules/redisha/manifests/sentinel.pp | 7 +++++- modules/stalwart/manifests/init.pp | 23 +++++++++++++++-- site/profiles/manifests/consul/client.pp | 7 +++++- site/profiles/manifests/consul/server.pp | 14 +++++++++-- site/profiles/manifests/dns/base.pp | 25 ++++++++++++++++--- site/profiles/manifests/dns/master.pp | 22 +++++++++++++--- site/profiles/manifests/etcd/node.pp | 9 +++++-- site/profiles/manifests/haproxy/dns.pp | 15 +++++------ site/profiles/manifests/minio/server.pp | 11 ++++++-- site/profiles/manifests/ntp/client.pp | 18 ++++++++++--- .../profiles/manifests/proxmox/clusterinit.pp | 11 +++++--- .../profiles/manifests/proxmox/clusterjoin.pp | 15 +++++------ site/profiles/manifests/sql/galera_member.pp | 7 +++++- site/profiles/manifests/sql/postgresdb.pp | 7 +++++- site/profiles/manifests/vault/server.pp | 7 +++++- 18 files changed, 169 insertions(+), 44 deletions(-) diff --git a/Puppetfile b/Puppetfile index 2934ad1..50bdcb5 100644 --- a/Puppetfile +++ b/Puppetfile @@ -53,7 +53,6 @@ mod 'saz-ssh', '13.1.0' mod 'saz-limits', '5.0.0' mod 'ghoneycutt-timezone', '4.0.0' mod 'ghoneycutt-puppet', '3.3.0' -mod 'dalen-puppetdbquery', '3.0.1' mod 'markt-galera', '3.1.0' mod 'kogitoapp-minio', '1.1.4' mod 'broadinstitute-certs', '3.0.1' diff --git a/modules/incus/manifests/cluster.pp b/modules/incus/manifests/cluster.pp index 80c76b5..a2cbd59 100644 --- a/modules/incus/manifests/cluster.pp +++ b/modules/incus/manifests/cluster.pp @@ -22,7 +22,12 @@ class incus::cluster ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/modules/redisha/manifests/redis.pp b/modules/redisha/manifests/redis.pp index cf375e4..309d0a6 100644 --- a/modules/redisha/manifests/redis.pp +++ b/modules/redisha/manifests/redis.pp @@ -20,7 +20,12 @@ class redisha::redis ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${redisha_members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/modules/redisha/manifests/sentinel.pp b/modules/redisha/manifests/sentinel.pp index 9ef4d42..f09c56a 100644 --- a/modules/redisha/manifests/sentinel.pp +++ b/modules/redisha/manifests/sentinel.pp @@ -23,7 +23,12 @@ class redisha::sentinel ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${redisha_members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/modules/stalwart/manifests/init.pp b/modules/stalwart/manifests/init.pp index 093e960..2bde921 100644 --- a/modules/stalwart/manifests/init.pp +++ b/modules/stalwart/manifests/init.pp @@ -167,7 +167,13 @@ class stalwart ( # Query cluster members for validation $cluster_query = "enc_role='${cluster_role}' and country='${facts['country']}' and region='${facts['region']}'" - $cluster_members_raw = query_nodes($cluster_query, 'networking.fqdn') + $cluster_members_raw = puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${cluster_role}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] } $cluster_members = $cluster_members_raw ? { undef => [], default => $cluster_members_raw, @@ -180,7 +186,20 @@ class stalwart ( # Query HAProxy nodes for proxy trusted networks $haproxy_query = "enc_role='${haproxy_role}' and country='${facts['country']}' and region='${facts['region']}'" - $haproxy_members_raw = query_nodes($haproxy_query, 'networking.ip') + $haproxy_members_raw = puppetdb_query( + "facts[certname,value] { + name = 'networking' and + certname in facts[certname] { + name = 'enc_role' and value = '${haproxy_role}' + } and + certname in facts[certname] { + name = 'country' and value = '${facts['country']}' + } and + certname in facts[certname] { + name = 'region' and value = '${facts['region']}' + } + }" + ).map |$fact| { $fact['value']['ip'] } $haproxy_ips = $haproxy_members_raw ? { undef => [], default => sort($haproxy_members_raw), diff --git a/site/profiles/manifests/consul/client.pp b/site/profiles/manifests/consul/client.pp index f79496f..a66a4ec 100644 --- a/site/profiles/manifests/consul/client.pp +++ b/site/profiles/manifests/consul/client.pp @@ -28,7 +28,12 @@ class profiles::consul::client ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${::facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/site/profiles/manifests/consul/server.pp b/site/profiles/manifests/consul/server.pp index c2107a4..039dea1 100644 --- a/site/profiles/manifests/consul/server.pp +++ b/site/profiles/manifests/consul/server.pp @@ -65,12 +65,22 @@ class profiles::consul::server ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${::facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) if $join_remote_regions { # get all nodes in the members_role for each other region $region_to_servers = $remote_regions.reduce({}) |$memo, $region| { - $servers = sort(query_nodes("enc_role='${members_role}' and region='${region}'", 'networking.fqdn')) + $servers = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${region}' } + }" + ).map |$fact| { $fact['certname'] }) $memo + { $region => $servers } } diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp index ee2fff0..0e5e51f 100644 --- a/site/profiles/manifests/dns/base.pp +++ b/site/profiles/manifests/dns/base.pp @@ -18,9 +18,28 @@ class profiles::dns::base ( $nameserver_array = $ns_role ? { undef => $nameservers, default => $use_ns ? { - 'all' => query_nodes("enc_role='${ns_role}'", 'networking.ip'), - 'region' => query_nodes("enc_role='${ns_role}' and region=${facts['region']}", 'networking.ip'), - 'country' => query_nodes("enc_role='${ns_role}' and country=${facts['country']}", 'networking.ip'), + 'all' => puppetdb_query( + "facts[certname,value] { + name = 'networking' and + certname in nodes[certname] { facts.enc_role = '${ns_role}' } + }" + ).map |$fact| { $fact['value']['ip'] }, + 'region' => puppetdb_query( + "facts[certname,value] { + name = 'networking' and + certname in nodes[certname] { + facts.enc_role = '${ns_role}' and facts.region = '${facts['region']}' + } + }" + ).map |$fact| { $fact['value']['ip'] }, + 'country' => puppetdb_query( + "facts[certname,value] { + name = 'networking' and + certname in nodes[certname] { + facts.enc_role = '${ns_role}' and facts.country = '${facts['country']}' + } + }" + ).map |$fact| { $fact['value']['ip'] }, } } diff --git a/site/profiles/manifests/dns/master.pp b/site/profiles/manifests/dns/master.pp index 73c23b6..4d92a6e 100644 --- a/site/profiles/manifests/dns/master.pp +++ b/site/profiles/manifests/dns/master.pp @@ -20,9 +20,21 @@ class profiles::dns::master ( $nameservers_array = $ns_role ? { undef => [$facts['networking']['fqdn']], default => $use_ns ? { - 'all' => sort(query_nodes("enc_role='${ns_role}'", 'networking.fqdn')), - 'region' => sort(query_nodes("enc_role='${ns_role}' and region=${facts['region']}", 'networking.fqdn')), - 'country' => sort(query_nodes("enc_role='${ns_role}' and country=${facts['country']}", 'networking.fqdn')), + 'all' => sort(puppetdb_query( + "facts[certname] { name = 'enc_role' and value = '${ns_role}' }" + ).map |$fact| { $fact['certname'] }), + 'region' => sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${ns_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }), + 'country' => sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${ns_role}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } + }" + ).map |$fact| { $fact['certname'] }), } } @@ -32,7 +44,9 @@ class profiles::dns::master ( $facts['networking']['fqdn'] => $facts['networking']['ip'] }, default => $nameservers_array.reduce({}) |$acc, $fqdn| { - $result = query_nodes("networking.fqdn='${fqdn}'", 'networking.ip') + $result = puppetdb_query( + "facts[certname,value] { name = 'networking' and certname = '${fqdn}' }" + ).map |$fact| { $fact['value']['ip'] } $ip = $result[0] $acc + { "${fqdn}." => $ip } } diff --git a/site/profiles/manifests/etcd/node.pp b/site/profiles/manifests/etcd/node.pp index 0a13f60..8eea4b6 100644 --- a/site/profiles/manifests/etcd/node.pp +++ b/site/profiles/manifests/etcd/node.pp @@ -18,7 +18,12 @@ class profiles::etcd::node ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ @@ -31,7 +36,7 @@ class profiles::etcd::node ( $initial_cluster = $servers_array.map |$fqdn| { # lookup the ip address for the current fqdn - $ip = query_nodes("networking.fqdn='${fqdn}'", 'networking.ip')[0] + $ip = puppetdb_query("facts[certname,value] { name = 'networking' and certname = '${fqdn}' }").map |$fact| { $fact['value']['ip'] }[0] # construct the string for this server "${fqdn}=https://${ip}:${peer_port}" diff --git a/site/profiles/manifests/haproxy/dns.pp b/site/profiles/manifests/haproxy/dns.pp index 721f5f7..8338ea2 100644 --- a/site/profiles/manifests/haproxy/dns.pp +++ b/site/profiles/manifests/haproxy/dns.pp @@ -30,13 +30,14 @@ class profiles::haproxy::dns ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes( - "enc_role='${facts['enc_role']}' and - country='${facts['country']}' and - region='${facts['region']}' and - environment='${facts['environment']}'", - 'networking.fqdn' - )) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${facts['enc_role']}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } and + certname in facts[certname] { name = 'environment' and value = '${facts['environment']}' } + }" + ).map |$fact| { $fact['certname'] }) # give enough time for a few hosts to be provisioned if length($servers_array) >= 3 { diff --git a/site/profiles/manifests/minio/server.pp b/site/profiles/manifests/minio/server.pp index 3fdec43..79f1b55 100644 --- a/site/profiles/manifests/minio/server.pp +++ b/site/profiles/manifests/minio/server.pp @@ -98,8 +98,15 @@ class profiles::minio::server ( } # if it is, find hosts, sort them so they dont cause changes every run - #$servers_array = sort(query_nodes("enc_role='${minio_members_role}'", 'networking.fqdn')) - $servers_array = sort(query_nodes("enc_role='${minio_members_role}' and minio_region='${minio_region}'", 'networking.fqdn')) + #$servers_array = sort(puppetdb_query( + # "facts[certname] { name = 'enc_role' and value = '${minio_members_role}' }" + #).map |$fact| { $fact['certname'] }) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${minio_members_role}' and + certname in facts[certname] { name = 'minio_region' and value = '${minio_region}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/site/profiles/manifests/ntp/client.pp b/site/profiles/manifests/ntp/client.pp index 516ad20..eae0e77 100644 --- a/site/profiles/manifests/ntp/client.pp +++ b/site/profiles/manifests/ntp/client.pp @@ -26,9 +26,21 @@ class profiles::ntp::client ( $ntpserver_array = $ntp_role ? { undef => $peers, default => $use_ntp ? { - 'all' => query_nodes("enc_role='${ntp_role}'", 'networking.fqdn'), - 'region' => query_nodes("enc_role='${ntp_role}' and region=${facts['region']}", 'networking.fqdn'), - 'country' => query_nodes("enc_role='${ntp_role}' and country=${facts['country']}", 'networking.fqdn'), + 'all' => puppetdb_query( + "facts[certname] { name = 'enc_role' and value = '${ntp_role}' }" + ).map |$fact| { $fact['certname'] }, + 'region' => puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${ntp_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }, + 'country' => puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${ntp_role}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } + }" + ).map |$fact| { $fact['certname'] }, } } diff --git a/site/profiles/manifests/proxmox/clusterinit.pp b/site/profiles/manifests/proxmox/clusterinit.pp index 65189b3..6c2851e 100644 --- a/site/profiles/manifests/proxmox/clusterinit.pp +++ b/site/profiles/manifests/proxmox/clusterinit.pp @@ -24,10 +24,13 @@ class profiles::proxmox::clusterinit { } } - $servers_array = sort(query_nodes( - "enc_role='${membersrole}' and country='${facts['country']}' and region='${facts['region']}'", - 'networking.fqdn' - )) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${membersrole}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) if ! $profiles::proxmox::params::pve_clusterinit_master { if !empty($servers_array) { diff --git a/site/profiles/manifests/proxmox/clusterjoin.pp b/site/profiles/manifests/proxmox/clusterjoin.pp index 7ab3ea5..2d0327d 100644 --- a/site/profiles/manifests/proxmox/clusterjoin.pp +++ b/site/profiles/manifests/proxmox/clusterjoin.pp @@ -11,13 +11,14 @@ class profiles::proxmox::clusterjoin { $root_password = $profiles::proxmox::params::root_password # query puppetdb for list of cluster members - $members_array = sort(query_nodes( - "enc_role='${membersrole}' and \ - country='${facts['country']}' and \ - region='${facts['region']}' and \ - pve_cluster.cluster_name='${clustername}'", - 'networking.fqdn' - )) + $members_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${membersrole}' and + certname in facts[certname] { name = 'country' and value = '${facts['country']}' } and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } and + certname in facts[certname] { name = 'pve_cluster' and value.cluster_name = '${clustername}' } + }" + ).map |$fact| { $fact['certname'] }) # check if the pve kernerl is running if $facts['kernelrelease'] == $profiles::proxmox::params::pve_kernel_release { diff --git a/site/profiles/manifests/sql/galera_member.pp b/site/profiles/manifests/sql/galera_member.pp index 843e2a3..797d2c8 100644 --- a/site/profiles/manifests/sql/galera_member.pp +++ b/site/profiles/manifests/sql/galera_member.pp @@ -48,7 +48,12 @@ class profiles::sql::galera_member ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${galera_members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${galera_members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/site/profiles/manifests/sql/postgresdb.pp b/site/profiles/manifests/sql/postgresdb.pp index f06b2ed..bb21825 100644 --- a/site/profiles/manifests/sql/postgresdb.pp +++ b/site/profiles/manifests/sql/postgresdb.pp @@ -18,7 +18,12 @@ class profiles::sql::postgresdb ( } # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ diff --git a/site/profiles/manifests/vault/server.pp b/site/profiles/manifests/vault/server.pp index 1e3a7cd..665b2ea 100644 --- a/site/profiles/manifests/vault/server.pp +++ b/site/profiles/manifests/vault/server.pp @@ -29,7 +29,12 @@ class profiles::vault::server ( if $members_lookup and $members_role != undef { # if it is, find hosts, sort them so they dont cause changes every run - $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${::facts['region']}'", 'networking.fqdn')) + $servers_array = sort(puppetdb_query( + "facts[certname] { + name = 'enc_role' and value = '${members_role}' and + certname in facts[certname] { name = 'region' and value = '${::facts['region']}' } + }" + ).map |$fact| { $fact['certname'] }) # else use provided array from params }else{ -- 2.47.3