fix: thundering hurd (#435)

- started all puppet clients at the same time, resulting in thundering herd
- add a randomness timer of 10 minutes

Reviewed-on: #435

hieradata/os/AlmaLinux/AlmaLinux8.yaml

@@ -10,8 +10,8 @@ profiles::yum::global::repos:
     name: powertools
     descr: powertools repository
     target: /etc/yum.repos.d/powertools.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os
+    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   unkin:
     name: unkin

hieradata/os/AlmaLinux/AlmaLinux9.yaml

@@ -3,12 +3,28 @@
 crypto_policies::policy: 'DEFAULT:SHA1'
 
 profiles::yum::global::repos:
+  baseos:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/baseos-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/baseos-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  extras:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/extras-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/extras-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  appstream:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/appstream-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/appstream-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  highavailability:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/ha-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/ha-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
   crb:
     name: crb
     descr: crb repository
     target: /etc/yum.repos.d/crb.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/crb-daily/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/crb-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
     mirrorlist: absent
   unkin:
     name: unkin

hieradata/os/AlmaLinux/all_releases.yaml

@@ -23,36 +23,36 @@ profiles::yum::global::repos:
     name: baseos
     descr: baseos repository
     target: /etc/yum.repos.d/baseos.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os
+    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/BaseOS/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   extras:
     name: extras
     descr: extras repository
     target: /etc/yum.repos.d/extras.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/extras/%{facts.os.architecture}/os
+    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/extras/%{facts.os.architecture}/os
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/extras/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   appstream:
     name: appstream
     descr: appstream repository
     target: /etc/yum.repos.d/appstream.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os
+    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/AppStream/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   highavailability:
     name: highavailability
     descr: highavailability repository
     target: /etc/yum.repos.d/highavailability.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os
+    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/almalinux/%{facts.os.release.full}/HighAvailability/%{facts.os.architecture}/os/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
   epel:
     name: epel
     descr: epel repository
     target: /etc/yum.repos.d/epel.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/epel/%{facts.os.release.major}/Everything/%{facts.os.architecture}
+    baseurl: https://packagerepo.service.consul/epel/%{facts.os.release.major}/everything-daily/%{facts.os.architecture}/os/
-    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/epel/RPM-GPG-KEY-EPEL-%{facts.os.release.major}
+    gpgkey: https://packagerepo.service.consul/epel/%{facts.os.release.major}/everything-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-EPEL-%{facts.os.release.major}
     mirrorlist: absent
   unkinben:
     name: unkinben

hieradata/roles/infra/k8s/control.yaml

@@ -58,6 +58,12 @@ consul::services:
         tcp: "%{hiera('networking_loopback0_ip')}:9345"
         interval: '10s'
         timeout: '1s'
+      - id: 'rke2_server_ping_check'
+        name: 'rke2 Server Ping Check'
+        http: "https://%{hiera('networking_loopback0_ip')}:9345/ping"
+        interval: '10s'
+        timeout: '3s'
+        tls_skip_verify: true
 profiles::consul::client::node_rules:
   - resource: service
     segment: api-k8s

site/profiles/manifests/puppet/client.pp

@@ -13,6 +13,8 @@ class profiles::puppet::client (
   Boolean $show_diff          = true,
   Boolean $usecacheonfailure  = false,
   Integer $facts_soft_limit   = 4096,
+  Boolean $splay              = true,
+  Integer $splaylimit         = 600,
 ) {
 
   # dont manage puppet.conf if this is a puppetmaster

site/profiles/templates/puppet/client/puppet.conf.erb

@@ -12,3 +12,5 @@ runtimeout = <%= @runtimeout %>
 show_diff = <%= @show_diff %>
 usecacheonfailure = <%= @usecacheonfailure %>
 number_of_facts_soft_limit = <%= @facts_soft_limit %>
+splay = <%= @splay %>
+splaylimit = <%= @splaylimit %>