From cf25a20a92da7573aa6a35df48d861442bfc2a1a Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 20 Sep 2026 00:32:40 +1000 Subject: [PATCH 01/13] Install encapic ENC client on puppet masters (#525) The VM masters classify through the cobbler ENC while the k8s compilers already use encapi. Install the client ahead of that cutover; external_nodes still points at cobbler-enc, so classification is unchanged. - pin the encapic package to 0.2.0 via profiles::packages::include - add profiles::puppet::encapic managing /etc/encapic/encapic.conf from a hiera-driven ENCAPI_URL, ordered after Package['encapic'] so the config is written once the RPM that owns the path is installed - include the class from profiles::puppet::puppetmaster Requires encapic 0.2.0 in the rpm-internal repo. Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/525 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/roles/infra/puppet/master.yaml | 9 ++++++ site/profiles/manifests/puppet/encapic.pp | 32 +++++++++++++++++++ .../profiles/manifests/puppet/puppetmaster.pp | 1 + 3 files changed, 42 insertions(+) create mode 100644 site/profiles/manifests/puppet/encapic.pp diff --git a/hieradata/roles/infra/puppet/master.yaml b/hieradata/roles/infra/puppet/master.yaml index 7673440..da9809b 100644 --- a/hieradata/roles/infra/puppet/master.yaml +++ b/hieradata/roles/infra/puppet/master.yaml @@ -26,6 +26,15 @@ profiles::puppet::cobbler_enc::packages: - 'requests' - 'PyYAML' profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-enc.git + +# encapic is installed alongside the cobbler ENC; external_nodes still points +# at cobbler-enc. Deep-merged with the entries in roles/infra/puppet.yaml. +profiles::packages::include: + encapic: + ensure: '0.2.0' + +profiles::puppet::encapic::encapi_url: https://encapi.k8s.syd1.au.unkin.net + profiles::puppet::r10k::r10k_repo: https://git.unkin.net/unkin/puppet-r10k.git profiles::puppet::g10k::bin_path: '/usr/bin/g10k' profiles::puppet::g10k::cfg_path: '/etc/puppetlabs/r10k/r10k.yaml' diff --git a/site/profiles/manifests/puppet/encapic.pp b/site/profiles/manifests/puppet/encapic.pp new file mode 100644 index 0000000..ecb0ed5 --- /dev/null +++ b/site/profiles/manifests/puppet/encapic.pp @@ -0,0 +1,32 @@ +# Class: profiles::puppet::encapic +# +# Manages the configuration for the encapic ENC client. The package itself is +# installed through profiles::packages (pinned in hiera); this class owns the +# config so the encapi endpoint can change without repackaging. +class profiles::puppet::encapic ( + Stdlib::HTTPUrl $encapi_url, + Stdlib::AbsolutePath $config_dir = '/etc/encapic', + String $config_name = 'encapic.conf', + String $owner = 'root', + String $group = 'root', +) { + + # The RPM ships this file as %config(noreplace), so puppet must write it only + # once the package is present or the install overwrites it. + file { $config_dir: + ensure => directory, + mode => '0755', + owner => $owner, + group => $group, + require => Package['encapic'], + } + + file { "${config_dir}/${config_name}": + ensure => file, + mode => '0644', + owner => $owner, + group => $group, + content => "ENCAPI_URL=${encapi_url}\n", + require => File[$config_dir], + } +} diff --git a/site/profiles/manifests/puppet/puppetmaster.pp b/site/profiles/manifests/puppet/puppetmaster.pp index 35d326d..dea779d 100644 --- a/site/profiles/manifests/puppet/puppetmaster.pp +++ b/site/profiles/manifests/puppet/puppetmaster.pp @@ -12,6 +12,7 @@ class profiles::puppet::puppetmaster ( include profiles::puppet::g10k include profiles::puppet::enc include profiles::puppet::cobbler_enc + include profiles::puppet::encapic include profiles::puppet::autosign include profiles::puppet::gems include profiles::helpers::certmanager From 1a907467e9a4b17fc0380bd4b356d174e4a179c7 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 20 Sep 2026 00:55:19 +1000 Subject: [PATCH 02/13] Shorten metadata_expire on internal RPM repos (#526) A package pinned in hieradata right after its RPM lands in artifactapi is invisible to dnf until the host's 1h cached metadata expires, so the first Puppet run after a release cannot find the version. - Set `metadata_expire` 60s on `rpm-internal`/`rpm-vendor` and their per-release variants for AlmaLinux and Fedora - Leave upstream mirrors on the 1h default - Drop the stale expiry note in `profiles::dns::updater` Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/526 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/os/AlmaLinux/all_releases.yaml | 4 ++++ hieradata/os/Fedora/all_releases.yaml | 2 ++ site/profiles/manifests/dns/updater.pp | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hieradata/os/AlmaLinux/all_releases.yaml b/hieradata/os/AlmaLinux/all_releases.yaml index 5b5320e..3ff5da7 100644 --- a/hieradata/os/AlmaLinux/all_releases.yaml +++ b/hieradata/os/AlmaLinux/all_releases.yaml @@ -77,6 +77,7 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' rpm-vendor: name: rpm-vendor descr: rpm-vendor repository @@ -84,6 +85,7 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' # Per-release variants, resolved from the host's EL major version so el8 # hosts pull rpm-internal-el8/rpm-vendor-el8, el9 hosts el9, etc. rpm-internal-release: @@ -93,6 +95,7 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-el%{facts.os.release.major}/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' rpm-vendor-release: name: rpm-vendor-el%{facts.os.release.major} descr: rpm-vendor-el%{facts.os.release.major} repository @@ -100,6 +103,7 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-el%{facts.os.release.major}/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' # Additional repositories - default to absent, roles can override with ensure: present # FRRouting repositories diff --git a/hieradata/os/Fedora/all_releases.yaml b/hieradata/os/Fedora/all_releases.yaml index 5e05288..4470b0b 100644 --- a/hieradata/os/Fedora/all_releases.yaml +++ b/hieradata/os/Fedora/all_releases.yaml @@ -60,6 +60,7 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-f%{facts.os.release.major}/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' rpm-vendor: name: rpm-vendor-f%{facts.os.release.major} descr: rpm-vendor-f%{facts.os.release.major} repository @@ -67,3 +68,4 @@ profiles::yum::global::repos: baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-f%{facts.os.release.major}/ gpgcheck: false mirrorlist: absent + metadata_expire: '60' diff --git a/site/profiles/manifests/dns/updater.pp b/site/profiles/manifests/dns/updater.pp index 17d914d..7ee2641 100644 --- a/site/profiles/manifests/dns/updater.pp +++ b/site/profiles/manifests/dns/updater.pp @@ -24,8 +24,7 @@ class profiles::dns::updater ( Stdlib::AbsolutePath $config_dir = '/etc/dns-updater', Stdlib::AbsolutePath $master_basedir = lookup('profiles::dns::master::basedir'), # dns-updater daemon (replaces the dns-update shell script). 'latest' so hosts - # pick up new releases (e.g. the record filter); rpm-internal metadata_expire - # is 1h so this does not thrash. + # pick up new releases (e.g. the record filter). String $package_ensure = 'latest', Stdlib::AbsolutePath $api_socket = '/run/dns-updater/api.sock', String $resync = '10m', From 3370aff38f8f59019fd390a6af16903e30d86fa6 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 20 Sep 2026 22:42:43 +1000 Subject: [PATCH 03/13] Classify puppet masters through encapi (#527) The VM puppet masters are the last part of the classification path still calling Cobbler; the k8s compilers already classify through encapi and the encapic RPM is installed on all six masters. - Point `profiles::puppet::server::external_nodes` at `/usr/bin/encapic-enc` for `roles::infra::puppet::master`. - Drop the stale comment about external_nodes still using cobbler-enc. `profiles::puppet::cobbler_enc` stays in place so the revert is one hiera line. Depends on the encapic 0.2.0 install (#525). Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/527 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/roles/infra/puppet/master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hieradata/roles/infra/puppet/master.yaml b/hieradata/roles/infra/puppet/master.yaml index da9809b..a856556 100644 --- a/hieradata/roles/infra/puppet/master.yaml +++ b/hieradata/roles/infra/puppet/master.yaml @@ -27,13 +27,13 @@ profiles::puppet::cobbler_enc::packages: - 'PyYAML' profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-enc.git -# encapic is installed alongside the cobbler ENC; external_nodes still points -# at cobbler-enc. Deep-merged with the entries in roles/infra/puppet.yaml. +# Deep-merged with the entries in roles/infra/puppet.yaml. profiles::packages::include: encapic: ensure: '0.2.0' profiles::puppet::encapic::encapi_url: https://encapi.k8s.syd1.au.unkin.net +profiles::puppet::server::external_nodes: '/usr/bin/encapic-enc' profiles::puppet::r10k::r10k_repo: https://git.unkin.net/unkin/puppet-r10k.git profiles::puppet::g10k::bin_path: '/usr/bin/g10k' From f933660d3bb2f060c0047abea622be670b3e6a02 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 20 Sep 2026 23:08:20 +1000 Subject: [PATCH 04/13] Fetch agent ENC facts from encapi (#528) The enc_role and enc_env facts still resolve against Cobbler on every agent, the last Cobbler dependency in the classification path now that the master-side ENC runs encapic-enc. - Point the fact at https://encapi.k8s.syd1.au.unkin.net - Rename the module and its messages from Cobbler to encapi Cache file, TTL and fallback-to-cache failure behaviour are unchanged. Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/528 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- modules/libs/lib/facter/enc_direct_facts.rb | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/libs/lib/facter/enc_direct_facts.rb b/modules/libs/lib/facter/enc_direct_facts.rb index 3aec01b..bf0fed4 100644 --- a/modules/libs/lib/facter/enc_direct_facts.rb +++ b/modules/libs/lib/facter/enc_direct_facts.rb @@ -6,8 +6,8 @@ require 'net/http' require 'uri' require 'fileutils' -# CobblerENC module: Fetches ENC data from Cobbler, caches it, and provides structured facts. -module CobblerENC +# EncapiENC module: Fetches ENC data from encapi, caches it, and provides structured facts. +module EncapiENC CACHE_FILE = '/var/cache/puppet_enc.yaml' CACHE_TTL = 7 * 24 * 60 * 60 # 7 days in seconds @enc_data = nil # In-memory cache for the ENC response @@ -29,8 +29,8 @@ module CobblerENC File.write(CACHE_FILE, cache_data.to_yaml) end - def self.fetch_from_cobbler - uri = URI("http://cobbler.main.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}") + def self.fetch_from_encapi + uri = URI("https://encapi.k8s.syd1.au.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}") response = Net::HTTP.get_response(uri) raise "Failed to fetch ENC data. HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess) @@ -41,7 +41,7 @@ module CobblerENC def self.retrieve_enc_data return @enc_data if @enc_data - @enc_data = fetch_from_cobbler + @enc_data = fetch_from_encapi write_cache(@enc_data) @enc_data end @@ -49,26 +49,26 @@ module CobblerENC def self.fetch_enc_data retrieve_enc_data rescue StandardError => e - Facter.warn("Error retrieving Cobbler ENC data: #{e.message}") + Facter.warn("Error retrieving encapi ENC data: #{e.message}") @enc_data = read_cache return @enc_data unless @enc_data.empty? - raise 'No cached ENC data available and Cobbler is down.' + raise 'No cached ENC data available and encapi is unreachable.' end def self.enc_role - fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in Cobbler ENC response') + fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in encapi ENC response') end def self.enc_env - fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in Cobbler ENC response') + fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in encapi ENC response') end end Facter.add('enc_role') do - setcode { CobblerENC.enc_role } + setcode { EncapiENC.enc_role } end Facter.add('enc_env') do - setcode { CobblerENC.enc_env } + setcode { EncapiENC.enc_env } end From 734fcb8cf43dd1e9ac327a7389cc9ba0ce19e0cf Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 20 Sep 2026 23:32:16 +1000 Subject: [PATCH 05/13] Trust the estate CA when fetching ENC facts (#529) Facter runs under Puppet's vendored Ruby, which reads its own bundled CA file and never the system trust store. The ENC fact now fetches over HTTPS, so every node fails certificate verification and falls back to its cached value. - set ca_file on the request to the vaultca anchor bundle - keep VERIFY_PEER on, and fall through to the existing cache path when the anchor is absent Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/529 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- modules/libs/lib/facter/enc_direct_facts.rb | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/modules/libs/lib/facter/enc_direct_facts.rb b/modules/libs/lib/facter/enc_direct_facts.rb index bf0fed4..7fde380 100644 --- a/modules/libs/lib/facter/enc_direct_facts.rb +++ b/modules/libs/lib/facter/enc_direct_facts.rb @@ -3,6 +3,7 @@ require 'facter' require 'yaml' require 'net/http' +require 'openssl' require 'uri' require 'fileutils' @@ -10,6 +11,13 @@ require 'fileutils' module EncapiENC CACHE_FILE = '/var/cache/puppet_enc.yaml' CACHE_TTL = 7 * 24 * 60 * 60 # 7 days in seconds + # Facter runs under Puppet's vendored ruby, whose OpenSSL trusts only + # /opt/puppetlabs/puppet/ssl/cert.pem and never the system trust store, so the + # estate CA anchor profiles::pki::vaultca installs has to be named explicitly. + CA_BUNDLE_PATHS = [ + '/etc/pki/ca-trust/source/anchors/vaultcaroot.pem', + '/usr/local/share/ca-certificates/vaultcaroot.pem' + ].freeze @enc_data = nil # In-memory cache for the ENC response def self.read_cache @@ -29,9 +37,22 @@ module EncapiENC File.write(CACHE_FILE, cache_data.to_yaml) end + def self.ca_bundle + CA_BUNDLE_PATHS.find { |path| File.exist?(path) } + end + + def self.http_client(uri) + client = Net::HTTP.new(uri.host, uri.port) + client.use_ssl = true + client.verify_mode = OpenSSL::SSL::VERIFY_PEER + bundle = ca_bundle + client.ca_file = bundle if bundle + client + end + def self.fetch_from_encapi uri = URI("https://encapi.k8s.syd1.au.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}") - response = Net::HTTP.get_response(uri) + response = http_client(uri).request(Net::HTTP::Get.new(uri)) raise "Failed to fetch ENC data. HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess) From cb9f8870bfda20fcc8329b16a03d00a6e76f936b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Thu, 24 Sep 2026 22:46:30 +1000 Subject: [PATCH 06/13] Trust the sshca host CA alongside the legacy signer (#530) Catalog compilation moved to the k8s puppetserver compilers, which sign host certificates against the terraform-managed `sshca` mount. Clients only trust the legacy `ssh-host-signer` CA, so every re-signed node (ausyd1nxvm2120 already) presents a certificate nothing accepts, and knownhosts emits no plain host-key fallback. - Add a second `@cert-authority *` entry for the `sshca` public key to `profiles::ssh::knownhosts::lines`. - Keep the legacy entry untouched so legacy-signed hosts still verify. Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/530 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/common.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index a0939e5..b23fc87 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -367,6 +367,7 @@ ssh::server::options: profiles::ssh::knownhosts::lines: - '@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1HD97vYxLTniE4qNpGuftUlvmkEXIuX8+7nbENv/IzsGUghEDRtyThjQ7ojNKIsQ7f8wXr0gMcI+fAPfrbcOMHCAoYMomikwL0b3h95SZI40q3CyM+0DMnwiVVDX6C1QxkO2Rv9cszSkCa85NotJhXiUuTBI9BFcRPy+mAhbpAru+bfypYofI0wW97XNTl8Jgwmni5MgutBIQAokFIn5ux8iWxndCH3AqDtmkwC5DfQeQ+wZx7rkwqJEpJffQzrjb1gIM6P9hDCVBBVPh/3o80IJ69rFWrJAZUb+JpG4cXJH0NcSW+wqc3JCT/x3q8VlHwOTXSlNNKtOJCRx73mB8e1XTTy2a9FgpKDDg5XQXWHAViJDz1RTRL9gRefMylRgKz4bXoTuY9kJWM8hPTyUejtukbJThlBJc3OmDxBZBF7F0iqB11pHexok43OCEiANodVa36eWu9/5X032Vm48fZ1/akDPY/NSy3wAn7kwut+A0/JAHFHASrq+1mt9YurkJegI+YHXO6eEWpBIpmI7ORHJbGL4MhkHrxYzVamuP8CkU7tXzsv138+wpOcRHNp9yJY4PT40BZkRf/O3O+jt3pj9Dj8rvgywF2W6hFzywh3Y78upOprRkQlQtHfsI8EyrYI8/hUw2u3H+3yPXh3YjWfqvWVG1BRLRHBV7m90uaw==' + - '@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDi80G0GtKMdRj4azPwxbJW46NG0y8seSVSnvFm2Ka/nckdUb/3lRlQuPYf1tkbqqFlcXjDM8+u0tzM2kLsgfn0Dpujm1fuoA66yGGweBjtxLFErH5+6+/KND5I9w8LMY2AtVztnBk/CVx8RwgrooABDRZiBH7OOAOpJqqFI7LvEsv61zC0nAqbYJ8uxfx+r4lJ9dUhK1woitEqC4npSRUn5KJK+KAEd7AzcUkZb6TO9A3oRPz1nQ/qU+QNMmVUi+wRj9kJR18mxErugyLLTNcFDBqSdHNun3eUNBUmoS2cAa8ZVscOLzbUGejVK9UY06Q7wu+J34Fzl8CN5tBqRHGZ3ykqGZ6gG+O0Egr9Rm3Obgkujpio2uTh27LhBy72vjgJ8kTFmPlzANTJFpKlsy91usSFPh8WZeIo6VpPLqnC32rjfNkfqHyPAeJ3+rdOkUZoDjMoZc3wxxLZTgMU5ud1w4LxQNRkNiaT/tRRNQF8o+pygkS7xxKduBBMzYdRS1OifaS4gyvP82oOyquWywhyFNtG7ph8FQwc34puM7FyxfaJ0XL/+zAfPMhDoOojvofADJo73R+FgVyer+mCJw4KtWJ4JzZrNTYz1Xl8WlhF8RDzOYfSH46qzJFa9FcRqgP4LUkgzdvcQ+1hBFpvpHtw3vl3DiqtZDDbK9SyrEtdnw==' profiles::base::groups::local: admins: From 1c01b7e6caa24ed81f488858f44295a9f82f805e Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 27 Sep 2026 09:55:58 +1000 Subject: [PATCH 07/13] Allow k8s edge health checks on arrstack hosts (#532) The Kubernetes-hosted haproxy edge probes the arr/nzbget backends on `/consul/health`, but its traffic arrives SNATed from the node ranges rather than the DMZ edge, so nginx returns 403 and every backend health-checks down. - Allow `198.18.21.0/24` (2.5gbe physical), `198.18.15.0/24` and `198.18.19.0/24` (node loopbacks) on the `arrstack_web_healthcheck` location. - Keep `198.18.24.0/24` so the existing DMZ edge stays healthy through cutover. --------- Co-authored-by: unkin-agent Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/532 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/roles/apps/media.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hieradata/roles/apps/media.yaml b/hieradata/roles/apps/media.yaml index 2780ede..6f2e113 100644 --- a/hieradata/roles/apps/media.yaml +++ b/hieradata/roles/apps/media.yaml @@ -65,6 +65,9 @@ profiles::nginx::simpleproxy::locations: - 127.0.0.1 - "%{facts.networking.ip}" - 198.18.24.0/24 + - 198.18.21.0/24 + - 198.18.15.0/24 + - 198.18.19.0/24 location_deny: - all # authorised access from external From c0e65fade3a62ddc6e21d305f4e34cee2acb1254 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 27 Sep 2026 10:01:49 +1000 Subject: [PATCH 08/13] Ship estate journald logs to the k8s log ingest endpoint (#531) The estate ships journald to the VM VictoriaLogs cluster, which is being left to age out rather than grow. New log capacity lands in k8s, so clients need to point there while the VM cluster keeps serving historical queries until its retention lapses. - repoint victorialogs::client::journald::inserturl at https://logs-ingest.k8s.syd1.au.unkin.net/insert/journald VMs already trust the issuing CA via the system bundle, so journal-upload needs no TLS change. Requires the k8s VLCluster deployed and serving /insert/journald first. Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/531 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index b23fc87..8ac63f9 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -401,7 +401,7 @@ networking::route_defaults: # logging: victorialogs::client::journald::enable: true -victorialogs::client::journald::inserturl: https://vlinsert.service.consul:9428/insert/journald +victorialogs::client::journald::inserturl: https://logs-ingest.k8s.syd1.au.unkin.net/insert/journald # FIXME these are for the proxmox ceph cluster profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8 From ec74484d890e9b33a05ee0ed97d26f59e9b248f4 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 27 Sep 2026 11:22:07 +1000 Subject: [PATCH 09/13] Pin the journald ingest URL to port 443 (#533) Estate journald ingestion has been down since 00:33Z. `systemd-journal-upload` appends `:19532/upload` to the configured URL whenever that URL carries no explicit port, so the portless k8s endpoint became `/insert/journald:19532/upload`, which vlinsert rejects as an unsupported path. The previous consul URL only worked because `:9428` was explicit. - pin `victorialogs::client::journald::inserturl` to port 443 Requests then land on `/insert/journald/upload`, which vlinsert accepts. Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/533 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 8ac63f9..02a6d8b 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -401,7 +401,7 @@ networking::route_defaults: # logging: victorialogs::client::journald::enable: true -victorialogs::client::journald::inserturl: https://logs-ingest.k8s.syd1.au.unkin.net/insert/journald +victorialogs::client::journald::inserturl: https://logs-ingest.k8s.syd1.au.unkin.net:443/insert/journald # FIXME these are for the proxmox ceph cluster profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8 From 410a1f13d0cfc9cb198c7c71967c1106f54a4b40 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 3 Oct 2026 20:46:44 +1000 Subject: [PATCH 10/13] Add 198.18.2.0/24 router loopback subnet (#535) Router loopbacks in 198.18.2.0/24 (e.g. prodnxsr0020, 198.18.2.160) match no subnet entry, so they get unknown environment/region/country facts and are not autosigned. - add 198.18.2.0/24 to subnet_facts as prod/syd1/au/common - add 198.18.2.0/24 to puppet master autosign subnet_ranges Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/535 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/roles/infra/puppet/master.yaml | 1 + modules/libs/lib/facter/subnet_facts.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/hieradata/roles/infra/puppet/master.yaml b/hieradata/roles/infra/puppet/master.yaml index a856556..f3182c1 100644 --- a/hieradata/roles/infra/puppet/master.yaml +++ b/hieradata/roles/infra/puppet/master.yaml @@ -1,5 +1,6 @@ --- profiles::puppet::autosign::subnet_ranges: + - '198.18.2.0/24' - '198.18.13.0/24' - '198.18.14.0/24' - '198.18.15.0/24' diff --git a/modules/libs/lib/facter/subnet_facts.rb b/modules/libs/lib/facter/subnet_facts.rb index a0ed78a..0accee1 100644 --- a/modules/libs/lib/facter/subnet_facts.rb +++ b/modules/libs/lib/facter/subnet_facts.rb @@ -5,6 +5,7 @@ require 'ipaddr' # a class that creates facts based on the subnet class SubnetAttributes SUBNET_TO_ATTRIBUTES = { + '198.18.2.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' }, # router loopbacks '198.18.13.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' }, '198.18.14.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' }, '198.18.15.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' }, From 47e3bdc8f50dc5b44ed7bd24bccd8b70481ee64d Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 3 Oct 2026 22:40:01 +1000 Subject: [PATCH 11/13] Add router role for prodnxsr0020 (#536) prodnxsr0020 runs FRR/OSPF hand-configured; bring its routing config under puppet without touching interfaces, firewall or dnsmasq. - add roles::infra::network::router (base + frrouting + frr_exporter) - enable ip_forward and disable rp_filter via sysctl::base - add prodnxsr0020 OSPF config (dum0, dum1, bond0.201; src 198.18.21.160) - pin dns, consul and router-id to dum0 instead of the WAN-facing primary IP - listen sshd on 127.0.0.1 and dum0 only, knocking out the common WAN primary IP - keep resolv.conf on the local dnsmasq (127.0.0.1) Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/536 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- .../nodes/prodnxsr0020.main.unkin.net.yaml | 45 +++++++++++++++++++ hieradata/roles/infra/network/router.yaml | 26 +++++++++++ site/roles/manifests/infra/network/router.pp | 12 +++++ 3 files changed, 83 insertions(+) create mode 100644 hieradata/nodes/prodnxsr0020.main.unkin.net.yaml create mode 100644 hieradata/roles/infra/network/router.yaml create mode 100644 site/roles/manifests/infra/network/router.pp diff --git a/hieradata/nodes/prodnxsr0020.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0020.main.unkin.net.yaml new file mode 100644 index 0000000..4d58a76 --- /dev/null +++ b/hieradata/nodes/prodnxsr0020.main.unkin.net.yaml @@ -0,0 +1,45 @@ +--- +# primary interface is the WAN uplink; pin host identity to the dum0 loopback +networking_loopback0_ip: 198.18.2.160 +networking_loopback1_ip: 198.18.21.160 + +# dns: keep the local dnsmasq resolver +profiles::dns::base::nameservers: + - 127.0.0.1 +profiles::dns::base::search: + - main.unkin.net +profiles::dns::base::primary_interface: dum0 +profiles::dns::updater::deny_ranges: + - 198.18.199.0/24 + - 198.18.200.0/24 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 10.10.12.0/24 # wg0 + - 103.216.190.0/23 # wan uplink +profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}" + +# ssh: listen on localhost and dum0 only; knock out the common wan primary ip +lookup_options: + ssh::server::options: + merge: + strategy: deep + knockout_prefix: '--' +ssh::server::options: + ListenAddress: + - "--%{facts.networking.ip}" + - 127.0.0.1 + - "%{hiera('networking_loopback0_ip')}" +profiles::ssh::sign::principals: + - "%{hiera('networking_loopback0_ip')}" + +# frrouting +frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}" +frrouting::ospfd_interfaces: + dum0: + area: 0.0.0.0 + dum1: + area: 0.0.0.0 + bond0.201: + area: 0.0.0.0 +frrouting::ospf_preferred_source_enable: true +frrouting::ospf_preferred_source: "%{hiera('networking_loopback1_ip')}" diff --git a/hieradata/roles/infra/network/router.yaml b/hieradata/roles/infra/network/router.yaml new file mode 100644 index 0000000..91ef40b --- /dev/null +++ b/hieradata/roles/infra/network/router.yaml @@ -0,0 +1,26 @@ +--- +hiera_include: + - frrouting + - exporters::frr_exporter + +# routing +sysctl::base::values: + net.ipv4.ip_forward: + value: '1' + net.ipv4.conf.all.rp_filter: + value: '0' + net.ipv4.conf.default.rp_filter: + value: '0' + +# frrouting +exporters::frr_exporter::enable: true +frrouting::ospfd_redistribute: + - connected +frrouting::daemons: + ospfd: true + +# consul +profiles::consul::client::node_rules: + - resource: service + segment: frr_exporter + disposition: write diff --git a/site/roles/manifests/infra/network/router.pp b/site/roles/manifests/infra/network/router.pp new file mode 100644 index 0000000..3a48da7 --- /dev/null +++ b/site/roles/manifests/infra/network/router.pp @@ -0,0 +1,12 @@ +# roles::infra::network::router +# an ospf router; frr only, interfaces and firewall are managed outside puppet +# +class roles::infra::network::router { + if $facts['firstrun'] { + include profiles::defaults + include profiles::firstrun::init + }else{ + include profiles::defaults + include profiles::base + } +} From 1ed268acdae982ed357e6eb264c4f669263fa834 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 3 Oct 2026 22:40:46 +1000 Subject: [PATCH 12/13] Add dnsmasq module (#537) The router runs dnsmasq as a DNS forwarder and DHCP relay from hand-edited config, so its state is not reproducible from code. - add `dnsmasq` module installing the package, rendering `/etc/dnsmasq.conf` from class params and running the service - cover listen addresses/interfaces, bind mode, upstream servers, no-resolv, cache-size, domain-needed, bogus-priv, per-domain forwards and dhcp-relay - add raw `options` lines for anything else - add `purge_config_dir` (default off) to remove unmanaged `/etc/dnsmasq.d` files Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/537 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- modules/dnsmasq/manifests/init.pp | 51 ++++++++++++++++++++++ modules/dnsmasq/templates/dnsmasq.conf.erb | 40 +++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 modules/dnsmasq/manifests/init.pp create mode 100644 modules/dnsmasq/templates/dnsmasq.conf.erb diff --git a/modules/dnsmasq/manifests/init.pp b/modules/dnsmasq/manifests/init.pp new file mode 100644 index 0000000..1fcfdd5 --- /dev/null +++ b/modules/dnsmasq/manifests/init.pp @@ -0,0 +1,51 @@ +# manage dnsmasq as a dns forwarder and dhcp relay +class dnsmasq ( + Boolean $manage_package = true, + Boolean $manage_service = true, + String $package_name = 'dnsmasq', + String $service_name = 'dnsmasq', + Stdlib::Absolutepath $config_file = '/etc/dnsmasq.conf', + Stdlib::Absolutepath $config_dir = '/etc/dnsmasq.d', + Boolean $purge_config_dir = false, + Array[String] $interfaces = [], + Array[Stdlib::IP::Address] $listen_addresses = ['127.0.0.1'], + Enum['bind-interfaces', 'bind-dynamic', 'none'] $bind_mode = 'bind-interfaces', + Boolean $no_resolv = false, + Array[String] $servers = [], + Hash[String, Array[String]] $forwards = {}, + Optional[Integer[0]] $cache_size = undef, + Boolean $domain_needed = true, + Boolean $bogus_priv = true, + Array[String] $dhcp_relays = [], + Array[String] $options = [], +) { + + if $manage_package { + package { $package_name: + ensure => installed, + before => File[$config_file, $config_dir], + } + } + + file { $config_dir: + ensure => directory, + recurse => $purge_config_dir, + purge => $purge_config_dir, + } + + file { $config_file: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => template('dnsmasq/dnsmasq.conf.erb'), + } + + if $manage_service { + service { $service_name: + ensure => running, + enable => true, + subscribe => File[$config_file, $config_dir], + } + } +} diff --git a/modules/dnsmasq/templates/dnsmasq.conf.erb b/modules/dnsmasq/templates/dnsmasq.conf.erb new file mode 100644 index 0000000..c5a09ae --- /dev/null +++ b/modules/dnsmasq/templates/dnsmasq.conf.erb @@ -0,0 +1,40 @@ +# THIS FILE IS MANAGED BY PUPPET +user=dnsmasq +group=dnsmasq +conf-dir=<%= @config_dir %>,.rpmnew,.rpmsave,.rpmorig + +<% @interfaces.each do |iface| -%> +interface=<%= iface %> +<% end -%> +<% unless @listen_addresses.empty? -%> +listen-address=<%= @listen_addresses.join(',') %> +<% end -%> +<% unless @bind_mode == 'none' -%> +<%= @bind_mode %> +<% end -%> +<% if @no_resolv -%> +no-resolv +<% end -%> +<% if @domain_needed -%> +domain-needed +<% end -%> +<% if @bogus_priv -%> +bogus-priv +<% end -%> +<% if @cache_size -%> +cache-size=<%= @cache_size %> +<% end -%> +<% @servers.each do |server| -%> +server=<%= server %> +<% end -%> +<% @forwards.keys.sort.each do |domain| -%> +<% @forwards[domain].each do |server| -%> +server=/<%= domain %>/<%= server %> +<% end -%> +<% end -%> +<% @dhcp_relays.each do |relay| -%> +dhcp-relay=<%= relay %> +<% end -%> +<% @options.each do |line| -%> +<%= line %> +<% end -%> From 02721045044a3319f0489879b06646f8542eb8b5 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 4 Oct 2026 14:17:07 +1100 Subject: [PATCH 13/13] Add wireguard module (#538) WireGuard on the router is configured by hand, so its tunnels are not reproducible from code. This adds a module to manage it from hieradata. - add `wireguard` class to install wireguard-tools and manage interfaces from a hash - add `wireguard::interface` to render `/etc/wireguard/.conf` (0600) and enable `wg-quick@` - keep private and preshared keys `Sensitive` end to end (`wireguard::interfaces` lookup_options `convert_to: Sensitive`, typed peer Struct) - without `private_key`, generate `/etc/wireguard/.key` (0600) only if absent and load it via PostUp, so the key never rotates - apply config changes with `wg syncconf` instead of restarting the tunnel Reviewed-on: https://git.unkin.net/unkin/puppet-prod/pulls/538 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- hieradata/common.yaml | 2 + modules/wireguard/manifests/init.pp | 44 +++++++++++++++++ modules/wireguard/manifests/interface.pp | 63 ++++++++++++++++++++++++ modules/wireguard/templates/wg.conf.erb | 31 ++++++++++++ 4 files changed, 140 insertions(+) create mode 100644 modules/wireguard/manifests/init.pp create mode 100644 modules/wireguard/manifests/interface.pp create mode 100644 modules/wireguard/templates/wg.conf.erb diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 02a6d8b..e29de34 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -178,6 +178,8 @@ lookup_options: convert_to: Sensitive stalwart::fallback_admin_password: convert_to: Sensitive + wireguard::interfaces: + convert_to: Sensitive facts_path: '/opt/puppetlabs/facter/facts.d' diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp new file mode 100644 index 0000000..a4301b6 --- /dev/null +++ b/modules/wireguard/manifests/init.pp @@ -0,0 +1,44 @@ +# manage wireguard interfaces via wg-quick +class wireguard ( + Boolean $manage_package = true, + String $package_name = 'wireguard-tools', + Variant[Hash, Sensitive[Hash]] $interfaces = {}, +) { + + if $manage_package { + package { $package_name: + ensure => installed, + before => File['/etc/wireguard'], + } + } + + file { '/etc/wireguard': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0700', + } + + # hiera hands eyaml secrets over as plain strings inside the (Sensitive) hash; re-wrap them per resource + $raw = $interfaces ? { + Sensitive => $interfaces.unwrap, + default => $interfaces, + } + + $raw.each |String $iface, Hash $data| { + $peers = $data.get('peers', []).map |Hash $peer| { + $peer['preshared_key'] =~ String ? { + true => $peer + { 'preshared_key' => Sensitive($peer['preshared_key']) }, + default => $peer, + } + } + $private_key = $data['private_key'] =~ String ? { + true => Sensitive($data['private_key']), + default => $data['private_key'], + } + + wireguard::interface { $iface: + * => $data + { 'peers' => $peers, 'private_key' => $private_key }, + } + } +} diff --git a/modules/wireguard/manifests/interface.pp b/modules/wireguard/manifests/interface.pp new file mode 100644 index 0000000..7428941 --- /dev/null +++ b/modules/wireguard/manifests/interface.pp @@ -0,0 +1,63 @@ +# manage one wg-quick interface; without private_key, /etc/wireguard/.key is generated once and loaded via PostUp +define wireguard::interface ( + Array[Stdlib::IP::Address] $addresses, + Optional[Stdlib::Port] $listen_port = undef, + Optional[Integer[1280, 9000]] $mtu = undef, + Optional[Sensitive[String[1]]] $private_key = undef, + Array[Struct[{ + public_key => String[1], + allowed_ips => Variant[String[1], Array[String[1], 1]], + preshared_key => Optional[Sensitive[String[1]]], + endpoint => Optional[String[1]], + persistent_keepalive => Optional[Integer[0, 65535]], + }]] $peers = [], +) { + + $conf = "/etc/wireguard/${name}.conf" + $key = $private_key.then |$k| { $k.unwrap } + + if $private_key =~ Undef { + $keyfile = "/etc/wireguard/${name}.key" + + exec { "wireguard_genkey_${name}": + command => "/bin/sh -c 'umask 077; wg genkey > ${keyfile}'", + creates => $keyfile, + path => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'], + require => File['/etc/wireguard'], + } + + file { $keyfile: + ensure => file, + owner => 'root', + group => 'root', + mode => '0600', + require => Exec["wireguard_genkey_${name}"], + before => [File[$conf], Service["wg-quick@${name}"]], + } + } + + file { $conf: + ensure => file, + owner => 'root', + group => 'root', + mode => '0600', + content => Sensitive(template('wireguard/wg.conf.erb')), + show_diff => false, + notify => Exec["wireguard_syncconf_${name}"], + } + + service { "wg-quick@${name}": + ensure => running, + enable => true, + require => File[$conf], + } + + # syncconf applies peer/key changes without bouncing the tunnel; address/mtu changes need a manual restart + exec { "wireguard_syncconf_${name}": + command => "/bin/bash -c 'wg syncconf ${name} <(wg-quick strip ${name})'", + onlyif => "/usr/sbin/ip link show ${name}", + path => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'], + refreshonly => true, + require => Service["wg-quick@${name}"], + } +} diff --git a/modules/wireguard/templates/wg.conf.erb b/modules/wireguard/templates/wg.conf.erb new file mode 100644 index 0000000..01b45bd --- /dev/null +++ b/modules/wireguard/templates/wg.conf.erb @@ -0,0 +1,31 @@ +# THIS FILE IS MANAGED BY PUPPET +[Interface] +<% @addresses.each do |addr| -%> +Address = <%= addr %> +<% end -%> +<% if @listen_port -%> +ListenPort = <%= @listen_port %> +<% end -%> +<% if @mtu -%> +MTU = <%= @mtu %> +<% end -%> +<% if @key -%> +PrivateKey = <%= @key %> +<% else -%> +PostUp = wg set %i private-key /etc/wireguard/%i.key +<% end -%> +<% @peers.each do |peer| -%> + +[Peer] +PublicKey = <%= peer['public_key'] %> +<% if peer['preshared_key'] -%> +PresharedKey = <%= peer['preshared_key'].unwrap %> +<% end -%> +AllowedIPs = <%= Array(peer['allowed_ips']).join(', ') %> +<% if peer['endpoint'] -%> +Endpoint = <%= peer['endpoint'] %> +<% end -%> +<% if peer['persistent_keepalive'] -%> +PersistentKeepalive = <%= peer['persistent_keepalive'] %> +<% end -%> +<% end -%>