From 4a85c5feff895ed8970e63a53ad1f3db86772a3e Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 25 Jan 2025 20:15:05 +1100
Subject: [PATCH 01/89] Adding
 hieradata/node/ausyd1nxvm1070.main.unkin.net.yaml (#213)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/213
---
 hieradata/nodes/ausyd1nxvm1070.main.unkin.net.yaml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 hieradata/nodes/ausyd1nxvm1070.main.unkin.net.yaml

diff --git a/hieradata/nodes/ausyd1nxvm1070.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1070.main.unkin.net.yaml
new file mode 100644
index 0000000..4983f52
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm1070.main.unkin.net.yaml
@@ -0,0 +1,7 @@
+---
+networking::interfaces:
+  eth0:
+    ipaddress: 198.18.13.80
+networking::routes:
+  default:
+    gateway: 198.18.13.254
\ No newline at end of file

From ab7ce3bbfa25a2f843b8516bc722fa237d05a573 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 25 Jan 2025 20:15:20 +1100
Subject: [PATCH 02/89] Adding
 hieradata/node/ausyd1nxvm1071.main.unkin.net.yaml (#214)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/214
---
 hieradata/nodes/ausyd1nxvm1071.main.unkin.net.yaml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 hieradata/nodes/ausyd1nxvm1071.main.unkin.net.yaml

diff --git a/hieradata/nodes/ausyd1nxvm1071.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1071.main.unkin.net.yaml
new file mode 100644
index 0000000..0dffb7a
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm1071.main.unkin.net.yaml
@@ -0,0 +1,7 @@
+---
+networking::interfaces:
+  eth0:
+    ipaddress: 198.18.13.81
+networking::routes:
+  default:
+    gateway: 198.18.13.254
\ No newline at end of file

From afd3405c98b6a890e4cbf3a97387a68499c0bf38 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 26 Jan 2025 20:00:20 +1100
Subject: [PATCH 03/89] feat: add etcd module/role (#215)

- add etcd module
- add etcd role, profile and hieradata

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/215
---
 hieradata/common.yaml                   |   2 +
 hieradata/roles/infra/etcd/node.eyaml   |   2 +
 hieradata/roles/infra/etcd/node.yaml    |  60 +++++++++++++
 modules/etcd/manifests/init.pp          | 110 ++++++++++++++++++++++++
 modules/etcd/templates/etcd.service.erb |  17 ++++
 site/profiles/manifests/etcd/node.pp    |  58 +++++++++++++
 site/roles/manifests/infra/etcd/node.pp |  11 +++
 7 files changed, 260 insertions(+)
 create mode 100644 hieradata/roles/infra/etcd/node.eyaml
 create mode 100644 hieradata/roles/infra/etcd/node.yaml
 create mode 100644 modules/etcd/manifests/init.pp
 create mode 100644 modules/etcd/templates/etcd.service.erb
 create mode 100644 site/profiles/manifests/etcd/node.pp
 create mode 100644 site/roles/manifests/infra/etcd/node.pp

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 518df23..c1379c8 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -135,6 +135,8 @@ lookup_options:
   keepalived::vrrp_instance:
     merge:
       strategy: deep
+  profiles::etcd::node::initial_cluster_token:
+    convert_to: Sensitive
 
 facts_path: '/opt/puppetlabs/facter/facts.d'
 
diff --git a/hieradata/roles/infra/etcd/node.eyaml b/hieradata/roles/infra/etcd/node.eyaml
new file mode 100644
index 0000000..40ffd6b
--- /dev/null
+++ b/hieradata/roles/infra/etcd/node.eyaml
@@ -0,0 +1,2 @@
+---
+profiles::etcd::node::initial_cluster_token: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAhLyXszXUU6Dkiw9bEJTH0RXGaV2751NzvLH94i7QHfNukvOslF/kaDOA+FwqG06xSKSKo24Qyj4ewYA3BzhN8XLf2E9uW2LuDrUoA6aXUP2tYPqiTw8zmmgsVV5t7Y5PeNcleV3KmfcJZJKp33yGCKtGF7ggvNvnied5slO6E1BDkcVnqO7sdyI0MqSvsvH4IvEmeiSWAcBRBnwVLIwfn10frIvUg0fH4uZR7DASfO/HstYWKAEacz4xYBv74TtVVtYHlPvnVwC20YIYDMrgBsm3XngyWIQvruQCgyIkRzHjUKCpp76HpyEqzdJdEdaywkODYNOT6ab1B5uUu9WaMjBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBADXLPOqFHdnVgJW5+iXJYcgCDK1Eyr+RwvMA+3VszYALU5B6OCH5maplwC5aUgiQZ7ew==]
diff --git a/hieradata/roles/infra/etcd/node.yaml b/hieradata/roles/infra/etcd/node.yaml
new file mode 100644
index 0000000..ff0231b
--- /dev/null
+++ b/hieradata/roles/infra/etcd/node.yaml
@@ -0,0 +1,60 @@
+---
+hiera_include:
+  - profiles::etcd::node
+
+profiles::etcd::node::members_lookup: true
+profiles::etcd::node::members_role: roles::infra::etcd::node
+
+profiles::etcd::node::config:
+  data-dir: /data/etcd
+  client-cert-auth: false
+  client-transport-security:
+    cert-file: /etc/pki/tls/vault/certificate.crt
+    key-file: /etc/pki/tls/vault/private.key
+    client-cert-auth: false
+    auto-tls: false
+  peer-transport-security:
+    cert-file: /etc/pki/tls/vault/certificate.crt
+    key-file: /etc/pki/tls/vault/private.key
+    client-cert-auth: false
+    auto-tls: false
+  allowed-cn:
+  max-wals: 5
+  max-snapshots: 5
+  snapshot-count: 10000
+  heartbeat-interval: 100
+  election-timeout: 1000
+  cipher-suites: [
+    TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+    TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+  ]
+  tls-min-version: 'TLS1.2'
+  tls-max-version: 'TLS1.3'
+
+profiles::pki::vault::alt_names:
+  - etcd.service.consul
+  - etcd.query.consul
+  - "etcd.service.%{facts.country}-%{facts.region}.consul"
+
+profiles::ssh::sign::principals:
+  - etcd.query.consul
+  - etcd.service.consul
+  - etcd.service.%{facts.country}-%{facts.region}.consul
+
+consul::services:
+  etcd:
+    service_name: 'etcd'
+    tags:
+      - 'etcd'
+    address: "%{facts.networking.ip}"
+    port: 2379
+    checks:
+      - id: 'etcd_tcp_check'
+        name: 'ETCD TCP Check'
+        tcp: "%{facts.networking.ip}:2379"
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: etcd
+    disposition: write
diff --git a/modules/etcd/manifests/init.pp b/modules/etcd/manifests/init.pp
new file mode 100644
index 0000000..fdf7c56
--- /dev/null
+++ b/modules/etcd/manifests/init.pp
@@ -0,0 +1,110 @@
+# manage etcd
+class etcd (
+  Boolean $manage_user = true,
+  Boolean $manage_group = true,
+  Boolean $manage_package = true,
+  Boolean $manage_service = true,
+  String[1] $package_name = 'etcd',
+  String[1] $user = 'etcd',
+  String[1] $group = 'etcd',
+  Stdlib::Absolutepath $config_path = '/etc/etcd',
+  Stdlib::Absolutepath $config_file = "${config_path}/etcd.yaml",
+  Hash $config = { 'data-dir' => '/var/lib/etcd' },
+  Integer $max_open_files = 40000,
+) {
+  if downcase($facts['kernel']) != 'linux' {
+    fail("Module etcd only supports Linux, not ${facts['kernel']}")
+  }
+  if $facts['service_provider'] != 'systemd' {
+    fail('Module etcd only supported on systems using systemd')
+  }
+  if ! $config['data-dir'] {
+    fail('Module etcd requires data-dir be specified in config Hash')
+  }
+
+  if $manage_package {
+    package { $package_name:
+      ensure => installed,
+    }
+  }
+
+  if $manage_user {
+    user { 'etcd':
+      ensure     => 'present',
+      name       => $user,
+      forcelocal => true,
+      shell      => '/bin/false',
+      gid        => $group,
+      home       => $config['data-dir'],
+      managehome => false,
+      system     => true,
+      before     => Systemd::Unit_file['etcd.service'],
+    }
+  }
+  if $manage_group {
+    group { 'etcd':
+      ensure     => 'present',
+      name       => $group,
+      forcelocal => true,
+      system     => true,
+      before     => Systemd::Unit_file['etcd.service'],
+    }
+  }
+
+  mkdir::p { $config_path: }
+  mkdir::p { $config['data-dir']: }
+
+  file { $config_file:
+    ensure  => 'file',
+    owner   => $user,
+    group   => $group,
+    mode    => '0600',
+    content => to_yaml($config),
+    notify  => Systemd::Unit_file['etcd.service'],
+    require => Mkdir::P[$config_path],
+  }
+
+  file { 'etcd-data-dir':
+    ensure  => 'directory',
+    path    => $config['data-dir'],
+    owner   => $user,
+    group   => $group,
+    mode    => '0700',
+    notify  => Systemd::Unit_file['etcd.service'],
+    require => Mkdir::P[$config['data-dir']],
+  }
+
+  file { 'etcd-data-dir-wal.tmp':
+    ensure  => 'directory',
+    path    => "${config['data-dir']}/wal.tmp",
+    owner   => $user,
+    group   => $group,
+    mode    => '0700',
+    notify  => Systemd::Unit_file['etcd.service'],
+    require => File['etcd-data-dir'],
+  }
+
+  if $config['wal-dir'] {
+    mkdir::p { $config['wal-dir']: }
+    file { 'etcd-wal-dir':
+      ensure  => 'directory',
+      path    => $config['wal-dir'],
+      owner   => $user,
+      group   => $group,
+      mode    => '0700',
+      notify  => Systemd::Unit_file['etcd.service'],
+      require => Mkdir::P[$config['wal-dir']],
+    }
+  }
+
+  if $manage_service {
+    include ::systemd
+
+    systemd::unit_file { 'etcd.service':
+      content => template('etcd/etcd.service.erb'),
+      enable  => true,
+      active  => true,
+      require => Package[$package_name],
+    }
+  }
+}
diff --git a/modules/etcd/templates/etcd.service.erb b/modules/etcd/templates/etcd.service.erb
new file mode 100644
index 0000000..967a95d
--- /dev/null
+++ b/modules/etcd/templates/etcd.service.erb
@@ -0,0 +1,17 @@
+# DO NOT EDIT: This file is being managed by Puppet.
+[Unit]
+Description=etcd key-value store
+Documentation=https://github.com/etcd-io/etcd
+After=network.target
+
+[Service]
+User=<%= @user %>
+Group=<%= @group %>
+Type=notify
+ExecStart=/usr/bin/etcd --config-file <%= @config_file %>
+Restart=always
+RestartSec=10s
+LimitNOFILE=<%= @max_open_files %>
+
+[Install]
+WantedBy=multi-user.target
diff --git a/site/profiles/manifests/etcd/node.pp b/site/profiles/manifests/etcd/node.pp
new file mode 100644
index 0000000..0a13f60
--- /dev/null
+++ b/site/profiles/manifests/etcd/node.pp
@@ -0,0 +1,58 @@
+# manage the use of the etcd module
+class profiles::etcd::node (
+  Sensitive[String[1]] $initial_cluster_token,
+  Boolean $members_lookup   = false,
+  String  $members_role     = undef,
+  Array   $servers          = [],
+  Stdlib::Port $client_port = 2379,
+  Stdlib::Port $peer_port   = 2380,
+  Hash $config              = {},
+){
+
+  # if lookup is enabled
+  if $members_lookup {
+
+    # check that the role is also set
+    unless !($members_role == undef) {
+      fail("members_role must be provided for ${title} when members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+
+  # else use provided array from params
+  }else{
+    $servers_array = sort($servers)
+  }
+
+  if length($servers_array) >= 3 {
+
+    # construct the initial-cluster string
+    $initial_cluster = $servers_array.map |$fqdn| {
+
+      # lookup the ip address for the current fqdn
+      $ip = query_nodes("networking.fqdn='${fqdn}'", 'networking.ip')[0]
+
+      # construct the string for this server
+      "${fqdn}=https://${ip}:${peer_port}"
+    }.join(',')
+
+    $defaults = {
+      'data-dir'                    => '/var/lib/etcd',
+      'name'                        => $facts['networking']['fqdn'],
+      'listen-client-urls'          => "https://${facts['networking']['ip']}:${client_port}",
+      'listen-peer-urls'            => "https://${facts['networking']['ip']}:${peer_port}",
+      'advertise-client-urls'       => "https://${facts['networking']['ip']}:${client_port}",
+      'initial-advertise-peer-urls' => "https://${facts['networking']['ip']}:${peer_port}",
+      'initial-cluster-token'       => $initial_cluster_token.unwrap,
+      'initial-cluster'             => $initial_cluster,
+      'initial-cluster-state'       => 'new',
+    }
+
+    $merged_config = merge($defaults, $config)
+
+    class { 'etcd':
+      config => $merged_config,
+    }
+  }
+}
diff --git a/site/roles/manifests/infra/etcd/node.pp b/site/roles/manifests/infra/etcd/node.pp
new file mode 100644
index 0000000..39a1793
--- /dev/null
+++ b/site/roles/manifests/infra/etcd/node.pp
@@ -0,0 +1,11 @@
+# a role to deploy etcd
+class roles::infra::etcd::node {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

From 0222f5ec4a17a4f0fb2cc6f2bab64f920f4660fe Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 26 Jan 2025 20:05:18 +1100
Subject: [PATCH 04/89] feat: update consul etcd check (#216)

- check the health api endpoint

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/216
---
 hieradata/roles/infra/etcd/node.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/hieradata/roles/infra/etcd/node.yaml b/hieradata/roles/infra/etcd/node.yaml
index ff0231b..38e933d 100644
--- a/hieradata/roles/infra/etcd/node.yaml
+++ b/hieradata/roles/infra/etcd/node.yaml
@@ -49,11 +49,13 @@ consul::services:
     address: "%{facts.networking.ip}"
     port: 2379
     checks:
-      - id: 'etcd_tcp_check'
-        name: 'ETCD TCP Check'
-        tcp: "%{facts.networking.ip}:2379"
+      - id: 'etcd_http_health_check'
+        name: 'ETCD HTTP Health Check'
+        http: "https://%{facts.networking.ip}:2379/health"
+        method: 'GET'
         interval: '10s'
         timeout: '1s'
+        tls_skip_verify: true
 profiles::consul::client::node_rules:
   - resource: service
     segment: etcd

From 7c1d96bd22cdd965b660a6690822f1c18650221e Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 27 Jan 2025 12:59:59 +1100
Subject: [PATCH 05/89] feat: add k8s and docker repos (#217)

- add docker stable repos to packagerepo
- add k8s 1.32 to packagerepo

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/217
---
 hieradata/roles/infra/reposync/syncer.yaml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index 34f8961..4aa91b4 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -38,6 +38,27 @@ profiles::consul::client::node_rules:
 profiles::reposync::webserver::nginx_listen_mode: both
 profiles::reposync::webserver::nginx_cert_type: vault
 profiles::reposync::repos_list:
+  docker_stable_el8:
+    repository: 'stable'
+    description: 'Docker CE Stable EL8'
+    osname: 'docker'
+    release: 'el8'
+    baseurl: 'https://download.docker.com/linux/centos/8/x86_64/stable/'
+    gpgkey: 'https://download.docker.com/linux/centos/gpg'
+  docker_stable_el9:
+    repository: 'stable'
+    description: 'Docker CE Stable EL9'
+    osname: 'docker'
+    release: 'el9'
+    baseurl: 'https://download.docker.com/linux/centos/9/x86_64/stable/'
+    gpgkey: 'https://download.docker.com/linux/centos/gpg'
+  k8s_1.32:
+    repository: '1.32'
+    description: 'Kubernetes 1.32'
+    osname: 'k8s'
+    release: '1.32'
+    baseurl: 'https://pkgs.k8s.io/core:/stable:/v1.32/rpm/'
+    gpgkey: 'https://pkgs.k8s.io/core:/stable:/v1.32/rpm/repodata/repomd.xml.key'
   mariadb_11_2_el8:
     repository: 'el8'
     description: 'MariaDB 11.2'

From b981a6fb01947e2549b92cc3960cc9295751d1c8 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 9 Mar 2025 17:49:35 +1100
Subject: [PATCH 06/89] feat: enable nomad jobs to query dns (#218)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/218
---
 hieradata/roles/infra/dns/resolver.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index 5c0387a..a98a6de 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -10,6 +10,11 @@ profiles::dns::resolver::acls:
       - 198.18.15.0/24
       - 198.18.16.0/24
       - 198.18.17.0/24
+  acl-nomad-jobs:
+    addresses:
+      - 198.18.64.0/24
+      - 198.18.65.0/24
+      - 198.18.66.0/24
 
 profiles::dns::resolver::zones:
   8.10.10.in-addr.arpa-forward:
@@ -74,3 +79,4 @@ profiles::dns::resolver::views:
       - 20.10.10.in-addr.arpa-forward
     match_clients:
       - acl-main.unkin.net
+      - acl-nomad-jobs

From 8eb751e22fd12e914d1f6a79d9f5713c2a83578f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 12 Mar 2025 23:09:15 +1100
Subject: [PATCH 07/89] feat: change enc_* fact to read direct from cobbler
 (#219)

- change enc_role and enc_env to read direct from cobbler
- cleanup profiles::base::facts

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/219
---
 .reek.yml                                     |  5 ++
 modules/libs/lib/facter/enc_direct_facts.rb   | 74 +++++++++++++++++++
 modules/libs/lib/facter/enc_env.rb            | 13 ----
 modules/libs/lib/facter/enc_role.rb           | 13 ----
 site/profiles/manifests/base.pp               |  1 -
 site/profiles/manifests/base/facts.pp         | 39 ----------
 site/profiles/manifests/firstrun/init.pp      |  3 +-
 .../base/facts/custom_facts.yaml.erb          |  3 -
 .../profiles/templates/base/facts/enc_env.erb |  1 -
 .../templates/base/facts/enc_role.erb         |  1 -
 10 files changed, 80 insertions(+), 73 deletions(-)
 create mode 100644 modules/libs/lib/facter/enc_direct_facts.rb
 delete mode 100644 modules/libs/lib/facter/enc_env.rb
 delete mode 100644 modules/libs/lib/facter/enc_role.rb
 delete mode 100644 site/profiles/manifests/base/facts.pp
 delete mode 100644 site/profiles/templates/base/facts/custom_facts.yaml.erb
 delete mode 100644 site/profiles/templates/base/facts/enc_env.erb
 delete mode 100644 site/profiles/templates/base/facts/enc_role.erb

diff --git a/.reek.yml b/.reek.yml
index 5d9b3c5..26c981b 100644
--- a/.reek.yml
+++ b/.reek.yml
@@ -3,3 +3,8 @@
 detectors:
   FeatureEnvy:
     enabled: false
+  TooManyStatements:
+    enabled: false
+  UncommunicativeVariableName:
+    accept:
+      - e
diff --git a/modules/libs/lib/facter/enc_direct_facts.rb b/modules/libs/lib/facter/enc_direct_facts.rb
new file mode 100644
index 0000000..3aec01b
--- /dev/null
+++ b/modules/libs/lib/facter/enc_direct_facts.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'facter'
+require 'yaml'
+require 'net/http'
+require 'uri'
+require 'fileutils'
+
+# CobblerENC module: Fetches ENC data from Cobbler, caches it, and provides structured facts.
+module CobblerENC
+  CACHE_FILE = '/var/cache/puppet_enc.yaml'
+  CACHE_TTL = 7 * 24 * 60 * 60 # 7 days in seconds
+  @enc_data = nil # In-memory cache for the ENC response
+
+  def self.read_cache
+    return {} unless File.exist?(CACHE_FILE)
+
+    cache_data = YAML.safe_load(File.read(CACHE_FILE)) || {}
+    timestamp = cache_data.fetch('timestamp', 0)
+
+    return cache_data if Time.now.to_i - timestamp < CACHE_TTL
+
+    {}
+  end
+
+  def self.write_cache(enc_data)
+    FileUtils.mkdir_p(File.dirname(CACHE_FILE))
+    cache_data = enc_data.merge({ 'timestamp' => Time.now.to_i })
+    File.write(CACHE_FILE, cache_data.to_yaml)
+  end
+
+  def self.fetch_from_cobbler
+    uri = URI("http://cobbler.main.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}")
+    response = Net::HTTP.get_response(uri)
+
+    raise "Failed to fetch ENC data. HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+    YAML.safe_load(response.body) || {}
+  end
+
+  def self.retrieve_enc_data
+    return @enc_data if @enc_data
+
+    @enc_data = fetch_from_cobbler
+    write_cache(@enc_data)
+    @enc_data
+  end
+
+  def self.fetch_enc_data
+    retrieve_enc_data
+  rescue StandardError => e
+    Facter.warn("Error retrieving Cobbler ENC data: #{e.message}")
+    @enc_data = read_cache
+    return @enc_data unless @enc_data.empty?
+
+    raise 'No cached ENC data available and Cobbler is down.'
+  end
+
+  def self.enc_role
+    fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in Cobbler ENC response')
+  end
+
+  def self.enc_env
+    fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in Cobbler ENC response')
+  end
+end
+
+Facter.add('enc_role') do
+  setcode { CobblerENC.enc_role }
+end
+
+Facter.add('enc_env') do
+  setcode { CobblerENC.enc_env }
+end
diff --git a/modules/libs/lib/facter/enc_env.rb b/modules/libs/lib/facter/enc_env.rb
deleted file mode 100644
index 2975c45..0000000
--- a/modules/libs/lib/facter/enc_env.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-Facter.add('enc_env') do
-  setcode do
-    require 'yaml'
-    # Check if the YAML file exists
-    if File.exist?('/root/.cache/custom_facts.yaml')
-      data = YAML.load_file('/root/.cache/custom_facts.yaml')
-      # Use safe navigation to return 'enc_env' or nil
-      data&.dig('enc_env')
-    end
-  end
-end
diff --git a/modules/libs/lib/facter/enc_role.rb b/modules/libs/lib/facter/enc_role.rb
deleted file mode 100644
index 979b4bf..0000000
--- a/modules/libs/lib/facter/enc_role.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-Facter.add('enc_role') do
-  setcode do
-    require 'yaml'
-    # Check if the YAML file exists
-    if File.exist?('/root/.cache/custom_facts.yaml')
-      data = YAML.load_file('/root/.cache/custom_facts.yaml')
-      # Use safe navigation to return 'enc_role' or nil
-      data&.dig('enc_role')
-    end
-  end
-end
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index 7eec9ab..fd4188e 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -22,7 +22,6 @@ class profiles::base (
     # include the base profiles
     include profiles::base::repos
     include profiles::packages
-    include profiles::base::facts
     include profiles::base::motd
     include profiles::base::scripts
     include profiles::base::hosts
diff --git a/site/profiles/manifests/base/facts.pp b/site/profiles/manifests/base/facts.pp
deleted file mode 100644
index 5344d19..0000000
--- a/site/profiles/manifests/base/facts.pp
+++ /dev/null
@@ -1,39 +0,0 @@
-# a class to define some global facts
-class profiles::base::facts {
-
-  # The path where external facts are stored
-  $facts_d_path = '/opt/puppetlabs/facter/facts.d'
-
-  # Ensure the directory exists
-  file { $facts_d_path:
-    ensure => directory,
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0755',
-  }
-
-  # cleanup old facts files
-  $fact_list = [ 'enc_role', 'enc_env' ]
-  $fact_list.each | String $item | {
-    file { "${facts_d_path}/${item}.txt":
-      ensure  => absent,
-    }
-  }
-
-  # ensure the path to the custom store exists
-  file { '/root/.cache':
-    ensure => directory,
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0750',
-  }
-
-  # create the file that will be read
-  file { '/root/.cache/custom_facts.yaml':
-    ensure  => file,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0644',
-    content => template('profiles/base/facts/custom_facts.yaml.erb'),
-  }
-}
diff --git a/site/profiles/manifests/firstrun/init.pp b/site/profiles/manifests/firstrun/init.pp
index 3b33622..cdb70e7 100644
--- a/site/profiles/manifests/firstrun/init.pp
+++ b/site/profiles/manifests/firstrun/init.pp
@@ -8,8 +8,7 @@ class profiles::firstrun::init {
   include profiles::base::repos
   include profiles::firstrun::packages
 
-  # set the motd and base facts
-  include profiles::base::facts
+  # set the motd
   include profiles::base::motd
 
   # create the sysadmin account
diff --git a/site/profiles/templates/base/facts/custom_facts.yaml.erb b/site/profiles/templates/base/facts/custom_facts.yaml.erb
deleted file mode 100644
index e4b3895..0000000
--- a/site/profiles/templates/base/facts/custom_facts.yaml.erb
+++ /dev/null
@@ -1,3 +0,0 @@
----
-enc_role: <%= @enc_role[0] %>
-enc_env: <%= @enc_env %>
diff --git a/site/profiles/templates/base/facts/enc_env.erb b/site/profiles/templates/base/facts/enc_env.erb
deleted file mode 100644
index 7695e4d..0000000
--- a/site/profiles/templates/base/facts/enc_env.erb
+++ /dev/null
@@ -1 +0,0 @@
-enc_env=<%= @enc_env %>
diff --git a/site/profiles/templates/base/facts/enc_role.erb b/site/profiles/templates/base/facts/enc_role.erb
deleted file mode 100644
index d59acdf..0000000
--- a/site/profiles/templates/base/facts/enc_role.erb
+++ /dev/null
@@ -1 +0,0 @@
-enc_role=<%= @enc_role[0] %>

From a30924471317050a3f654bf2ba81c1c59a2eb596 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 13 Mar 2025 21:23:40 +1100
Subject: [PATCH 08/89] feat: add nomad nodes (#220)

- change existing nodes to be nomad-agents

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/220
---
 hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml | 10 +++++++++-
 hieradata/nodes/ausyd1nxvm1068.main.unkin.net.yaml | 10 +++++++++-
 hieradata/nodes/ausyd1nxvm1069.main.unkin.net.yaml | 10 +++++++++-
 hieradata/roles/infra/dns/resolver.yaml            |  3 +++
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml
index 7c0516c..4ec44f8 100644
--- a/hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm1067.main.unkin.net.yaml
@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.77
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.77
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
\ No newline at end of file
+    gateway: 198.18.13.254
+docker::bip: '198.18.67.254/24'
diff --git a/hieradata/nodes/ausyd1nxvm1068.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1068.main.unkin.net.yaml
index c52cba9..2089753 100644
--- a/hieradata/nodes/ausyd1nxvm1068.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm1068.main.unkin.net.yaml
@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.78
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.78
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
\ No newline at end of file
+    gateway: 198.18.13.254
+docker::bip: '198.18.68.254/24'
diff --git a/hieradata/nodes/ausyd1nxvm1069.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1069.main.unkin.net.yaml
index b802e58..6dc3c14 100644
--- a/hieradata/nodes/ausyd1nxvm1069.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm1069.main.unkin.net.yaml
@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.79
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.79
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
\ No newline at end of file
+    gateway: 198.18.13.254
+docker::bip: '198.18.69.254/24'
diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index a98a6de..f39588c 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -15,6 +15,9 @@ profiles::dns::resolver::acls:
       - 198.18.64.0/24
       - 198.18.65.0/24
       - 198.18.66.0/24
+      - 198.18.67.0/24
+      - 198.18.68.0/24
+      - 198.18.69.0/24
 
 profiles::dns::resolver::zones:
   8.10.10.in-addr.arpa-forward:

From e0c3a234246cd64cee0ef485a642caec961f3f0c Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 13 Mar 2025 21:48:47 +1100
Subject: [PATCH 09/89] fix: define missing .cache directory (#221)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/221
---
 site/profiles/manifests/firstrun/complete.pp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/site/profiles/manifests/firstrun/complete.pp b/site/profiles/manifests/firstrun/complete.pp
index b80d5a4..fabd873 100644
--- a/site/profiles/manifests/firstrun/complete.pp
+++ b/site/profiles/manifests/firstrun/complete.pp
@@ -1,6 +1,11 @@
 # profiles::firstrun::complete
 class profiles::firstrun::complete {
 
+  file {'/root/.cache':
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'root',
+  }
   file {'/root/.cache/puppet_firstrun_complete':
     ensure  => 'file',
     owner   => 'root',

From 771b981d910575d20973f05c281436ea709247a4 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 20 Mar 2025 19:21:40 +1100
Subject: [PATCH 10/89] feat: enable nomad to manage sessions/services (#222)

- this is required to start patroni

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/222
---
 hieradata/roles/infra/nomad/agent.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/hieradata/roles/infra/nomad/agent.yaml b/hieradata/roles/infra/nomad/agent.yaml
index f300b30..3c9d1c3 100644
--- a/hieradata/roles/infra/nomad/agent.yaml
+++ b/hieradata/roles/infra/nomad/agent.yaml
@@ -64,3 +64,9 @@ profiles::consul::client::node_rules:
   - resource: service_prefix
     segment: ''
     disposition: write
+  - resource: key_prefix
+    segment: "nomad"
+    disposition: write
+  - resource: session_prefix
+    segment: ""
+    disposition: write

From adc0cf2c093ab114c79089fe37949310c0d91b19 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 29 Mar 2025 19:40:01 +1100
Subject: [PATCH 11/89] neoloc/lxd_hosts (#223)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/223
---
 hieradata/nodes/prodnxsr0009.main.unkin.net.yaml | 10 ++++++++++
 hieradata/nodes/prodnxsr0010.main.unkin.net.yaml | 10 ++++++++++
 hieradata/nodes/prodnxsr0011.main.unkin.net.yaml | 10 ++++++++++
 hieradata/nodes/prodnxsr0012.main.unkin.net.yaml | 10 ++++++++++
 hieradata/nodes/prodnxsr0013.main.unkin.net.yaml | 10 ++++++++++
 modules/networking/manifests/init.pp             |  5 ++++-
 6 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/prodnxsr0013.main.unkin.net.yaml

diff --git a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
new file mode 100644
index 0000000..6a43056
--- /dev/null
+++ b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
@@ -0,0 +1,10 @@
+---
+networking::interfaces:
+  enp2s0:
+    ipaddress: 198.18.15.9
+  enp3s0:
+    ipaddress: 10.18.15.9
+    mtu: 9000
+networking::routes:
+  default:
+    gateway: 198.18.15.254
diff --git a/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
new file mode 100644
index 0000000..d9e7592
--- /dev/null
+++ b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
@@ -0,0 +1,10 @@
+---
+networking::interfaces:
+  enp2s0:
+    ipaddress: 198.18.15.10
+  enp3s0:
+    ipaddress: 10.18.15.10
+    mtu: 9000
+networking::routes:
+  default:
+    gateway: 198.18.15.254
diff --git a/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
new file mode 100644
index 0000000..28a45ab
--- /dev/null
+++ b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
@@ -0,0 +1,10 @@
+---
+networking::interfaces:
+  enp2s0:
+    ipaddress: 198.18.15.11
+  enp3s0:
+    ipaddress: 10.18.15.11
+    mtu: 9000
+networking::routes:
+  default:
+    gateway: 198.18.15.254
diff --git a/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb68c4
--- /dev/null
+++ b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
@@ -0,0 +1,10 @@
+---
+networking::interfaces:
+  enp2s0:
+    ipaddress: 198.18.15.12
+  enp3s0:
+    ipaddress: 10.18.15.12
+    mtu: 9000
+networking::routes:
+  default:
+    gateway: 198.18.15.254
diff --git a/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
new file mode 100644
index 0000000..221e494
--- /dev/null
+++ b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
@@ -0,0 +1,10 @@
+---
+networking::interfaces:
+  enp2s0:
+    ipaddress: 198.18.15.13
+  enp3s0:
+    ipaddress: 10.18.15.13
+    mtu: 9000
+networking::routes:
+  default:
+    gateway: 198.18.15.254
diff --git a/modules/networking/manifests/init.pp b/modules/networking/manifests/init.pp
index ae3970a..d2076ca 100644
--- a/modules/networking/manifests/init.pp
+++ b/modules/networking/manifests/init.pp
@@ -29,7 +29,10 @@ class networking (
 
   # determine which networking service to restart
   $restart_command = $facts['os']['family'] ? {
-    'RedHat' => '/usr/bin/systemctl restart network',
+    'RedHat' => $facts['os']['release']['major'] ? {
+      '8' => '/usr/bin/systemctl restart network',
+      '9' => '/usr/bin/systemctl restart NetworkManager',
+    },
     'Debian' => '/usr/bin/systemctl restart networking',
     default  => fail('Unsupported OS in networking-restart-command'),
   }

From b95bcbd10aa57120ce04ffed648102d46c4a9645 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 29 Mar 2025 20:08:31 +1100
Subject: [PATCH 12/89] feat: add zfs to reposync (#224)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/224
---
 hieradata/roles/infra/reposync/syncer.yaml | 28 ++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index 4aa91b4..2c1b63d 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -122,3 +122,31 @@ profiles::reposync::repos_list:
     release: 'rhel9'
     baseurl: 'https://download.postgresql.org/pub/repos/yum/16/redhat/rhel-9-x86_64/'
     gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  zfs_dkms_rhel8:
+    repository: 'dkms'
+    description: 'ZFS DKMS RHEL 8'
+    osname: 'zfs'
+    release: 'rhel8'
+    baseurl: 'http://download.zfsonlinux.org/epel/8/x86_64/'
+    gpgkey: 'https://packagerepo.service.consul/zfs/gpg/RPM-GPG-KEY-openzfs-2013'
+  zfs_kmod_rhel8:
+    repository: 'kmod'
+    description: 'ZFS KMOD RHEL 8'
+    osname: 'zfs'
+    release: 'rhel8'
+    baseurl: 'http://download.zfsonlinux.org/epel/8/kmod/x86_64/'
+    gpgkey: 'https://packagerepo.service.consul/zfs/gpg/RPM-GPG-KEY-openzfs-2013'
+  zfs_dkms_rhel9:
+    repository: 'dkms'
+    description: 'ZFS DKMS RHEL 9'
+    osname: 'zfs'
+    release: 'rhel9'
+    baseurl: 'http://download.zfsonlinux.org/epel/9/x86_64/'
+    gpgkey: 'https://packagerepo.service.consul/zfs/gpg/RPM-GPG-KEY-openzfs-2022'
+  zfs_kmod_rhel9:
+    repository: 'kmod'
+    description: 'ZFS KMOD RHEL 9'
+    osname: 'zfs'
+    release: 'rhel9'
+    baseurl: 'http://download.zfsonlinux.org/epel/9/kmod/x86_64/'
+    gpgkey: 'https://packagerepo.service.consul/zfs/gpg/RPM-GPG-KEY-openzfs-2022'

From d0eb4c078db626283248d8806978983a9cff0809 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 29 Mar 2025 22:31:02 +1100
Subject: [PATCH 13/89] feat: add zfs modules (#225)

- add zfs_core module to puppetfile (provides zfs/zpool provider)
- add module to manage zfs

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/225
---
 Puppetfile                                    |   1 +
 .../zfs/lib/facter/zfs_zpool_cache_present.rb |  14 ++
 modules/zfs/manifests/config.pp               |  10 ++
 modules/zfs/manifests/init.pp                 |  52 ++++++
 modules/zfs/manifests/install.pp              | 151 ++++++++++++++++++
 modules/zfs/manifests/service.pp              |  90 +++++++++++
 6 files changed, 318 insertions(+)
 create mode 100644 modules/zfs/lib/facter/zfs_zpool_cache_present.rb
 create mode 100644 modules/zfs/manifests/config.pp
 create mode 100644 modules/zfs/manifests/init.pp
 create mode 100644 modules/zfs/manifests/install.pp
 create mode 100644 modules/zfs/manifests/service.pp

diff --git a/Puppetfile b/Puppetfile
index 65c883d..7532e88 100644
--- a/Puppetfile
+++ b/Puppetfile
@@ -19,6 +19,7 @@ mod 'puppetlabs-haproxy', '8.0.0'
 mod 'puppetlabs-java', '10.1.2'
 mod 'puppetlabs-reboot', '5.0.0'
 mod 'puppetlabs-docker', '10.0.1'
+mod 'puppetlabs-zfs_core', '1.6.1'
 
 # puppet
 mod 'puppet-python', '7.0.0'
diff --git a/modules/zfs/lib/facter/zfs_zpool_cache_present.rb b/modules/zfs/lib/facter/zfs_zpool_cache_present.rb
new file mode 100644
index 0000000..d2e87fe
--- /dev/null
+++ b/modules/zfs/lib/facter/zfs_zpool_cache_present.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Facter.add('zfs_zpool_cache_present') do
+  confine kernel: 'Linux'
+  setcode do
+    File.exist?('/etc/zfs/zpool.cache')
+  end
+end
+
+Facter.add('zfs_zpool_cache_present') do
+  setcode do
+    false
+  end
+end
diff --git a/modules/zfs/manifests/config.pp b/modules/zfs/manifests/config.pp
new file mode 100644
index 0000000..b1e05fa
--- /dev/null
+++ b/modules/zfs/manifests/config.pp
@@ -0,0 +1,10 @@
+# manage zfs config
+class zfs::config {
+
+  file { $zfs::conf_dir:
+    ensure => directory,
+    owner  => 0,
+    group  => 0,
+    mode   => '0644',
+  }
+}
diff --git a/modules/zfs/manifests/init.pp b/modules/zfs/manifests/init.pp
new file mode 100644
index 0000000..6feeeed
--- /dev/null
+++ b/modules/zfs/manifests/init.pp
@@ -0,0 +1,52 @@
+# Installs basic ZFS kernel and userland support.
+#
+# @example Declaring the class
+#   include zfs
+#
+# @example Tuning the ZFS ARC
+#   class { 'zfs':
+#     zfs_arc_max => to_bytes('256 M'),
+#     zfs_arc_min => to_bytes('128 M'),
+#   }
+#
+# @param conf_dir Top-level configuration directory, usually `/etc/zfs`.
+# @param kmod_type Whether to use DKMS kernel packages or ones built to match
+#   the running kernel (only applies to RHEL platforms).
+# @param manage_repo Whether to setup and manage external package repositories.
+# @param package_name The name of the top-level metapackage that installs ZFS
+#   support.
+# @param service_manage Whether to manage the various ZFS services.
+# @param zfs_arc_max Maximum size of the ARC in bytes.
+# @param zfs_arc_min Minimum size of the ARC in bytes.
+class zfs (
+  Optional[Integer[0]]              $zfs_arc_max,
+  Optional[Integer[0]]              $zfs_arc_min,
+  Optional[Hash]                    $zpools,
+  Optional[Hash]                    $datasets,
+  Stdlib::Absolutepath              $conf_dir = '/etc/zfs',
+  Enum['dkms', 'kabi']              $kmod_type = 'kabi',
+  Boolean                           $manage_repo = true,
+  Variant[String, Array[String, 1]] $package_name = 'zfs',
+  Boolean                           $service_manage = true,
+) {
+
+  contain zfs::install
+  contain zfs::config
+  contain zfs::service
+
+  Class['zfs::install'] ~> Class['zfs::config'] ~> Class['zfs::service']
+
+  # create zpools
+  $zpools.each | $zpool, $data | {
+    zpool { $zpool:
+      * => $data
+    }
+  }
+
+  # create datasets
+  $datasets.each | $dataset, $data | {
+    zfs { $dataset:
+      * => $data
+    }
+  }
+}
diff --git a/modules/zfs/manifests/install.pp b/modules/zfs/manifests/install.pp
new file mode 100644
index 0000000..cb78576
--- /dev/null
+++ b/modules/zfs/manifests/install.pp
@@ -0,0 +1,151 @@
+# manage zfs install/repos
+class zfs::install {
+
+  if $zfs::manage_repo {
+    case $facts['os']['family'] {
+      'RedHat': {
+        $baseurl = 'http://download.zfsonlinux.org'
+        $release = $facts['os']['release']['major'] ? {
+          '6'     => '6',
+          '7'     => $facts['os']['release']['full'] ? {
+            /^7\.[012]/ => '7',
+            default     => regsubst($facts['os']['release']['full'], '^7\.(\d+).*$', '7.\1'),
+          },
+          '8'     => $facts['os']['release']['full'] ? {
+            /^8\.4/ => '8.3',
+            default => regsubst($facts['os']['release']['full'], '^8\.(\d+).*$', '8.\1'),
+          },
+          default => regsubst($facts['os']['release']['full'], '^(\d\.\d+).*$', '\1'),
+        }
+
+        yumrepo { 'zfs':
+          baseurl => "${baseurl}/epel/${release}/\$basearch/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - dkms",
+          enabled => Integer($zfs::kmod_type == 'dkms'),
+          before  => Package[$zfs::package_name],
+        }
+
+        yumrepo { 'zfs-kmod':
+          baseurl => "${baseurl}/epel/${release}/kmod/\$basearch/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - kmod",
+          enabled => Integer($zfs::kmod_type == 'kabi'),
+        }
+
+        yumrepo { 'zfs-source':
+          baseurl => "${baseurl}/epel/${release}/SRPMS/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - Source",
+          enabled => 0,
+        }
+
+        yumrepo { 'zfs-testing':
+          baseurl => "${baseurl}/epel-testing/${release}/\$basearch/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - dkms - Testing",
+          enabled => 0,
+        }
+
+        yumrepo { 'zfs-testing-kmod':
+          baseurl => "${baseurl}/epel-testing/${release}/kmod/\$basearch/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - kmod - Testing",
+          enabled => 0,
+        }
+
+        yumrepo { 'zfs-testing-source':
+          baseurl => "${baseurl}/epel-testing/${release}/SRPMS/",
+          descr   => "ZFS on Linux for EL${facts['os']['release']['major']} - Testing Source",
+          enabled => 0,
+        }
+      }
+      default: {
+        # noop
+      }
+    }
+  }
+
+  # Handle these dependencies separately as they shouldn't be guarded by
+  # `$zfs::manage_repo`
+  case $facts['os']['family'] {
+    'RedHat': {
+      case $zfs::kmod_type {
+        'dkms': {
+          # Puppet doesn't like managing multiple versions of the same package.
+          # By using the version in the name Yum will do the right thing
+          ensure_packages(["kernel-devel-${facts['kernelrelease']}"], {
+            ensure => present,
+            before => Package[$zfs::package_name],
+          })
+        }
+        default: {
+          # noop
+        }
+      }
+    }
+    'Debian': {
+      case $facts['os']['name'] {
+        'Ubuntu': {
+          # noop
+        }
+        default: {
+          ensure_packages(["linux-headers-${facts['kernelrelease']}", "linux-headers-${facts['os']['architecture']}"], {
+            before => Package[$zfs::package_name],
+          })
+        }
+      }
+    }
+    default: {
+      # noop
+    }
+  }
+
+  # This is to work around the broken Debian 9 packages. Upon install the
+  # zfs-mount.service is started first which is the only unit that doesn't
+  # have an "ExecStartPre=-/sbin/modprobe zfs" line so the package can never
+  # be installed!
+  if $facts['os']['name'] == 'Debian' and $facts['os']['release']['major'] == '9' {
+    exec { 'zfs systemctl daemon-reload':
+      command     => 'systemctl daemon-reload',
+      refreshonly => true,
+      path        => $facts['path'],
+    }
+
+    Exec['zfs systemctl daemon-reload'] -> Package[$zfs::package_name]
+
+    file { '/etc/systemd/system/zfs-mount.service.d':
+      ensure => directory,
+      owner  => 0,
+      group  => 0,
+      mode   => '0644',
+    }
+
+    file { '/etc/systemd/system/zfs-mount.service.d/override.conf':
+      ensure  => file,
+      owner   => 0,
+      group   => 0,
+      mode    => '0644',
+      content => @(EOS/L),
+        [Service]
+        ExecStartPre=-/sbin/modprobe zfs
+        | EOS
+      notify  => Exec['zfs systemctl daemon-reload'],
+    }
+  }
+
+  # These need to be done here so the kernel settings are present before the
+  # package is installed and potentially loading the kernel module
+  $config = delete_undef_values({
+    'zfs_arc_max' => $zfs::zfs_arc_max,
+    'zfs_arc_min' => $zfs::zfs_arc_min,
+  })
+
+  $config.each |$option,$value| {
+    kmod::option { "zfs ${option}":
+      module => 'zfs',
+      option => $option,
+      value  => $value,
+      before => Package[$zfs::package_name],
+    }
+  }
+
+  package { $zfs::package_name:
+    ensure => present,
+  }
+}
diff --git a/modules/zfs/manifests/service.pp b/modules/zfs/manifests/service.pp
new file mode 100644
index 0000000..64ec056
--- /dev/null
+++ b/modules/zfs/manifests/service.pp
@@ -0,0 +1,90 @@
+# manage zfs services
+class zfs::service {
+
+  if $zfs::service_manage {
+
+    exec { 'modprobe zfs':
+      path   => $facts['path'],
+      unless => 'grep -q "^zfs " /proc/modules',
+    }
+
+    case $facts['service_provider'] {
+      'systemd': {
+        $cache_ensure = str2bool($facts['zfs_zpool_cache_present']) ? {
+          true    => 'running',
+          default => 'stopped',
+        }
+
+        $scan_ensure = str2bool($facts['zfs_zpool_cache_present']) ? {
+          true    => 'stopped',
+          default => 'running',
+        }
+
+        service { 'zfs-import-cache':
+          ensure     => $cache_ensure,
+          enable     => true,
+          hasstatus  => true,
+          hasrestart => true,
+          require    => Exec['modprobe zfs'],
+          before     => Service['zfs-mount'],
+        }
+
+        service { 'zfs-import-scan':
+          ensure     => $scan_ensure,
+          enable     => true,
+          hasstatus  => true,
+          hasrestart => true,
+          require    => Exec['modprobe zfs'],
+          before     => Service['zfs-mount'],
+        }
+      }
+      default: {
+
+        case $facts['os']['family'] {
+          'RedHat': {
+            service { 'zfs-import':
+              ensure     => running,
+              enable     => true,
+              hasstatus  => true,
+              hasrestart => true,
+              require    => Exec['modprobe zfs'],
+              before     => Service['zfs-mount'],
+            }
+          }
+          'Debian': {
+            $import_ensure = str2bool($facts['zfs_zpool_cache_present']) ? {
+              true    => 'running',
+              default => 'stopped',
+            }
+
+            service { 'zpool-import':
+              ensure     => $import_ensure,
+              enable     => true,
+              hasstatus  => true,
+              hasrestart => true,
+              require    => Exec['modprobe zfs'],
+            }
+          }
+          default: {
+            # noop
+          }
+        }
+      }
+    }
+
+    service { 'zfs-mount':
+      ensure     => running,
+      enable     => true,
+      hasstatus  => true,
+      hasrestart => true,
+      before     => Service['zfs-share'],
+    }
+
+    service { 'zfs-share':
+      ensure     => running,
+      enable     => true,
+      hasstatus  => true,
+      hasrestart => true,
+    }
+  }
+}

From 3a4e606459c4b1044bdc2e40617ac7b0ca37ca36 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 29 Mar 2025 22:37:37 +1100
Subject: [PATCH 14/89] chore: set yum/dnf metadata expiry (#226)

- set expiry to 1 day so that dnf frequently checks for updates from packagerepo

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/226
---
 site/profiles/manifests/defaults.pp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/site/profiles/manifests/defaults.pp b/site/profiles/manifests/defaults.pp
index d86b76a..210a1b8 100644
--- a/site/profiles/manifests/defaults.pp
+++ b/site/profiles/manifests/defaults.pp
@@ -32,13 +32,14 @@ class profiles::defaults {
   }
 
   Yumrepo {
-    ensure     => 'present',
-    enabled    => 1,
-    gpgcheck   => 1,
-    require    => [
+    ensure          => 'present',
+    enabled         => 1,
+    gpgcheck        => 1,
+    metadata_expire => '1d',
+    require         => [
       Class['profiles::pki::vaultca'],
       Class['crypto_policies'],
     ],
-    notify     => Exec['dnf_makecache'],
+    notify          => Exec['dnf_makecache'],
   }
 }

From 4e4774507704f03818fd01892a4aa7fa33a89b4b Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 29 Mar 2025 22:50:08 +1100
Subject: [PATCH 15/89] chore: setup unkin repo for el9 and el8 (#227)

- update the unkin repo definition for el8 and el9

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/227
---
 hieradata/os/AlmaLinux/AlmaLinux8.yaml   | 8 ++++++++
 hieradata/os/AlmaLinux/AlmaLinux9.yaml   | 8 ++++++++
 hieradata/os/AlmaLinux/all_releases.yaml | 8 --------
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/hieradata/os/AlmaLinux/AlmaLinux8.yaml b/hieradata/os/AlmaLinux/AlmaLinux8.yaml
index 808275c..798fea7 100644
--- a/hieradata/os/AlmaLinux/AlmaLinux8.yaml
+++ b/hieradata/os/AlmaLinux/AlmaLinux8.yaml
@@ -13,3 +13,11 @@ profiles::yum::global::repos:
     baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/PowerTools/%{facts.os.architecture}/os
     gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
+  unkin:
+    name: unkin
+    descr: unkin repository
+    target: /etc/yum.repos.d/unkin.repo
+    baseurl: https://git.query.consul/api/packages/unkin/rpm/almalinux/el8
+    gpgkey: https://git.query.consul/api/packages/unkin/rpm/repository.key
+    gpgcheck: false
+    mirrorlist: absent
diff --git a/hieradata/os/AlmaLinux/AlmaLinux9.yaml b/hieradata/os/AlmaLinux/AlmaLinux9.yaml
index f275d86..7c98e9c 100644
--- a/hieradata/os/AlmaLinux/AlmaLinux9.yaml
+++ b/hieradata/os/AlmaLinux/AlmaLinux9.yaml
@@ -10,3 +10,11 @@ profiles::yum::global::repos:
     baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os
     gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
     mirrorlist: absent
+  unkin:
+    name: unkin
+    descr: unkin repository
+    target: /etc/yum.repos.d/unkin.repo
+    baseurl: https://git.query.consul/api/packages/unkin/rpm/almalinux/el9
+    gpgkey: https://git.query.consul/api/packages/unkin/rpm/repository.key
+    gpgcheck: false
+    mirrorlist: absent
diff --git a/hieradata/os/AlmaLinux/all_releases.yaml b/hieradata/os/AlmaLinux/all_releases.yaml
index db5a2e1..5a09a26 100644
--- a/hieradata/os/AlmaLinux/all_releases.yaml
+++ b/hieradata/os/AlmaLinux/all_releases.yaml
@@ -59,14 +59,6 @@ profiles::yum::global::repos:
     baseurl: https://packagerepo.service.consul/puppet7/el/%{facts.os.release.major}-daily/%{facts.os.architecture}/os/
     gpgkey: https://packagerepo.service.consul/puppet7/el/%{facts.os.release.major}-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-puppet-20250406
     mirrorlist: absent
-  unkin:
-    name: unkin
-    descr: unkin repository
-    target: /etc/yum.repos.d/unkin.repo
-    baseurl: https://git.query.consul/api/packages/unkin/rpm/almalinux/el8
-    gpgkey: https://git.query.consul/api/packages/unkin/rpm/repository.key
-    gpgcheck: false
-    mirrorlist: absent
   unkinben:
     name: unkinben
     descr: unkinben repository

From dd5a4646ffd168aae6f72fc915ca049406d08a15 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 00:51:49 +1100
Subject: [PATCH 16/89] feat: update all modules (#228)

- update puppetlabs-* modules
- update puppet-* modules
- add limits and sysctl

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/228
---
 Puppetfile                      | 77 +++++++++++++++++----------------
 hieradata/common.yaml           |  8 ++++
 site/profiles/manifests/base.pp |  4 ++
 3 files changed, 51 insertions(+), 38 deletions(-)

diff --git a/Puppetfile b/Puppetfile
index 7532e88..d38a073 100644
--- a/Puppetfile
+++ b/Puppetfile
@@ -2,54 +2,54 @@ forge 'forge.puppetlabs.com'
 moduledir 'external_modules'
 
 # puppetlabs
-mod 'puppetlabs-stdlib', '9.1.0'
-mod 'puppetlabs-inifile', '6.0.0'
-mod 'puppetlabs-concat', '9.0.0'
-mod 'puppetlabs-vcsrepo', '6.1.0'
-mod 'puppetlabs-yumrepo_core', '2.0.0'
-mod 'puppetlabs-apt', '9.4.0'
-mod 'puppetlabs-lvm', '2.1.0'
-mod 'puppetlabs-puppetdb', '7.13.0'
-mod 'puppetlabs-postgresql', '9.1.0'
-mod 'puppetlabs-firewall', '6.0.0'
-mod 'puppetlabs-accounts', '8.1.0'
-mod 'puppetlabs-mysql', '15.0.0'
+mod 'puppetlabs-stdlib', '9.7.0'
+mod 'puppetlabs-inifile', '6.2.0'
+mod 'puppetlabs-concat', '9.1.0'
+mod 'puppetlabs-vcsrepo', '7.0.0'
+mod 'puppetlabs-yumrepo_core', '2.1.0'
+mod 'puppetlabs-apt', '10.0.1'
+mod 'puppetlabs-lvm', '3.0.1'
+mod 'puppetlabs-puppetdb', '8.1.0'
+mod 'puppetlabs-postgresql', '10.5.0'
+mod 'puppetlabs-firewall', '8.1.4'
+mod 'puppetlabs-accounts', '8.2.2'
+mod 'puppetlabs-mysql', '16.2.0'
 mod 'puppetlabs-xinetd', '3.4.1'
-mod 'puppetlabs-haproxy', '8.0.0'
-mod 'puppetlabs-java', '10.1.2'
-mod 'puppetlabs-reboot', '5.0.0'
-mod 'puppetlabs-docker', '10.0.1'
-mod 'puppetlabs-zfs_core', '1.6.1'
+mod 'puppetlabs-haproxy', '8.2.0'
+mod 'puppetlabs-java', '11.1.0'
+mod 'puppetlabs-reboot', '5.1.0'
+mod 'puppetlabs-docker', '10.2.0'
 
 # puppet
-mod 'puppet-python', '7.0.0'
-mod 'puppet-systemd', '5.1.0'
-mod 'puppet-yum', '7.0.0'
-mod 'puppet-archive', '7.0.0'
-mod 'puppet-chrony', '2.6.0'
-mod 'puppet-puppetboard', '9.0.0'
-mod 'puppet-nginx', '5.0.0'
-mod 'puppet-selinux', '4.1.0'
-mod 'puppet-prometheus', '13.4.0'
-mod 'puppet-grafana', '13.1.0'
-mod 'puppet-consul', '8.0.0'
-mod 'puppet-vault', '4.1.0'
+mod 'puppet-python', '7.4.0'
+mod 'puppet-systemd', '8.1.0'
+mod 'puppet-yum', '7.2.0'
+mod 'puppet-archive', '7.1.0'
+mod 'puppet-chrony', '3.0.0'
+mod 'puppet-puppetboard', '11.0.0'
+mod 'puppet-nginx', '6.0.1'
+mod 'puppet-selinux', '5.0.0'
+mod 'puppet-prometheus', '16.0.0'
+mod 'puppet-grafana', '14.1.0'
+mod 'puppet-consul', '9.1.0'
+mod 'puppet-vault', '4.1.1'
 mod 'puppet-dhcp', '6.1.0'
 mod 'puppet-keepalived', '5.1.0'
-mod 'puppet-extlib', '7.0.0'
-mod 'puppet-network', '2.2.0'
-mod 'puppet-kmod', '4.0.1'
+mod 'puppet-extlib', '7.5.1'
+mod 'puppet-network', '2.2.1'
+mod 'puppet-kmod', '4.1.0'
 mod 'puppet-filemapper', '4.0.0'
-mod 'puppet-letsencrypt', '11.0.0'
-mod 'puppet-rundeck', '9.1.0'
-mod 'puppet-redis', '11.0.0'
+mod 'puppet-letsencrypt', '11.1.0'
+mod 'puppet-rundeck', '9.2.0'
+mod 'puppet-redis', '11.1.0'
 mod 'puppet-nodejs', '11.0.0'
 
 # other
-mod 'ghoneycutt-puppet', '3.3.0'
-mod 'saz-sudo', '8.0.0'
-mod 'saz-ssh', '12.1.0'
+mod 'saz-sudo', '9.0.2'
+mod 'saz-ssh', '13.1.0'
+mod 'saz-limits', '5.0.0'
 mod 'ghoneycutt-timezone', '4.0.0'
+mod 'ghoneycutt-puppet', '3.3.0'
 mod 'dalen-puppetdbquery', '3.0.1'
 mod 'markt-galera', '3.1.0'
 mod 'kogitoapp-minio', '1.1.4'
@@ -59,6 +59,7 @@ mod 'h0tw1r3-gitea', '3.2.0'
 mod 'rehan-mkdir', '2.0.0'
 mod 'tailoredautomation-patroni', '2.0.0'
 mod 'ssm-crypto_policies', '0.3.3'
+mod 'thias-sysctl', '1.0.8'
 
 mod 'bind',
   :git => 'https://git.service.au-syd1.consul/unkinben/puppet-bind.git',
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index c1379c8..ece85d6 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -137,6 +137,12 @@ lookup_options:
       strategy: deep
   profiles::etcd::node::initial_cluster_token:
     convert_to: Sensitive
+  sysctl::base::values:
+    merge:
+      strategy: deep
+  limits::entries:
+    merge:
+      strategy: deep
 
 facts_path: '/opt/puppetlabs/facter/facts.d'
 
@@ -145,6 +151,8 @@ hiera_include:
   - networking
   - ssh::server
   - profiles::accounts::rundeck
+  - limits
+  - sysctl::base
 
 profiles::ntp::client::ntp_role: 'roles::infra::ntp::server'
 profiles::ntp::client::use_ntp: 'region'
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index fd4188e..c1a4feb 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -56,6 +56,10 @@ class profiles::base (
       include profiles::qemu::agent
     }
 
+    class { 'limits':
+      purge_limits_d_dir => false,
+    }
+
     # include classes from hiera
     $hiera_include = lookup('hiera_include', Array[String], 'unique', [])
     $hiera_exclude = lookup('hiera_exclude', Array[String], 'unique', [])

From 6a04701891940952c6aa4f126aae536cb4f3c3e8 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 00:56:04 +1100
Subject: [PATCH 17/89] feat: add incus role (#229)

- add basic infra::incus role
- add autossl, consul and ssh-principals for incus

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/229
---
 hieradata/roles/infra/incus/node.yaml    | 33 ++++++++++++++++++++++++
 site/roles/manifests/infra/incus/node.pp | 10 +++++++
 2 files changed, 43 insertions(+)
 create mode 100644 hieradata/roles/infra/incus/node.yaml
 create mode 100644 site/roles/manifests/infra/incus/node.pp

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
new file mode 100644
index 0000000..a0c8ecb
--- /dev/null
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -0,0 +1,33 @@
+---
+profiles::pki::vault::alt_names:
+  - incus.service.consul
+  - incus.query.consul
+  - "incus.service.%{facts.country}-%{facts.region}.consul"
+
+profiles::ssh::sign::principals:
+  - incus.service.consul
+  - incus.query.consul
+  - "incus.service.%{facts.country}-%{facts.region}.consul"
+
+# configure consul service
+consul::services:
+  incus:
+    service_name: 'incus'
+    tags:
+      - 'incus'
+      - 'container'
+      - 'lxd'
+    address: "%{facts.networking.ip}"
+    port: 8443
+    checks:
+      - id: 'incus_https_check'
+        name: 'incus HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:8443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: incus
+    disposition: write
diff --git a/site/roles/manifests/infra/incus/node.pp b/site/roles/manifests/infra/incus/node.pp
new file mode 100644
index 0000000..070bbf1
--- /dev/null
+++ b/site/roles/manifests/infra/incus/node.pp
@@ -0,0 +1,10 @@
+# a role to deploy a incus node
+class roles::infra::incus::node {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}

From bdf420973d7c5aab9dc212cff4391a00ae3babad Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 01:12:53 +1100
Subject: [PATCH 18/89] feat: add incus module (#230)

- add a basic incus module

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/230
---
 modules/incus/lib/facter/incus.rb             | 18 ++++++
 modules/incus/manifests/cluster.pp            | 57 +++++++++++++++++++
 modules/incus/manifests/init.pp               | 35 ++++++++++++
 modules/incus/templates/join_preseed.yaml.erb | 20 +++++++
 4 files changed, 130 insertions(+)
 create mode 100644 modules/incus/lib/facter/incus.rb
 create mode 100644 modules/incus/manifests/cluster.pp
 create mode 100644 modules/incus/manifests/init.pp
 create mode 100644 modules/incus/templates/join_preseed.yaml.erb

diff --git a/modules/incus/lib/facter/incus.rb b/modules/incus/lib/facter/incus.rb
new file mode 100644
index 0000000..e9639f6
--- /dev/null
+++ b/modules/incus/lib/facter/incus.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+Facter.add(:incus) do
+  setcode do
+    # Check if the 'incus' executable exists
+    incus_path = Facter::Util::Resolution.which('incus')
+    next {} unless incus_path # Return an empty fact if incus isn't found
+
+    # Run the `incus info` command using the found path
+    incus_output = Facter::Core::Execution.execute("#{incus_path} info")
+    next {} if incus_output.empty? # Return an empty fact if there's no output
+
+    # Parse the output as YAML and return it
+    YAML.safe_load(incus_output)
+  end
+end
diff --git a/modules/incus/manifests/cluster.pp b/modules/incus/manifests/cluster.pp
new file mode 100644
index 0000000..80c76b5
--- /dev/null
+++ b/modules/incus/manifests/cluster.pp
@@ -0,0 +1,57 @@
+# manage incus clusters
+class incus::cluster (
+  Boolean $members_lookup   = false,
+  String  $members_role     = undef,
+  String  $master           = undef,
+  Array   $servers          = [],
+  Stdlib::Fqdn $server_fqdn = $facts['networking']['fqdn'],
+  Stdlib::Port $server_port = 8443,
+){
+
+  # check that the master is named
+  unless !($master == undef) {
+    fail("master must be provided for ${title}")
+  }
+
+  # if lookup is enabled
+  if $members_lookup {
+
+    # check that the role is also set
+    unless !($members_role == undef) {
+      fail("members_role must be provided for ${title} when members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+
+  # else use provided array from params
+  }else{
+    $servers_array = $servers
+  }
+
+  # if its not an empty array. Give puppetdb a chance to be populated with data.
+  if length($servers_array) >= 3 {
+
+    # check if this is the master_node
+    if $master == $trusted['certname'] {
+      $master_bool = true
+    }else{
+      $master_bool = false
+    }
+
+    # find bootstrap status for servers
+    $bootstrap_array = puppetdb_query("inventory[certname, facts] { facts.enc_role = '${members_role}' }").map |$node| {
+      {
+        'fqdn'        => $node['certname'],
+        'ip'          => $node['facts']['networking']['ip'],
+        'clustered'   => $node['facts']['incus']['environment']['server_clustered'],
+        'certificate' => $node['facts']['incus']['environment']['certificate'],
+      }
+    }
+
+    # determine if the cluster is bootstrapped
+    $cluster_bootstrapped = $bootstrap_array.any |$server| {
+      $server['fqdn'] == $master and $server['clustered'] == true
+    }
+  }
+}
diff --git a/modules/incus/manifests/init.pp b/modules/incus/manifests/init.pp
new file mode 100644
index 0000000..0e7dc19
--- /dev/null
+++ b/modules/incus/manifests/init.pp
@@ -0,0 +1,35 @@
+class incus (
+  Array[String] $packages   = [
+    'incus',
+    'incus-tools',
+    'incus-client'
+  ],
+) {
+
+  package { $packages:
+    ensure => installed,
+  }
+
+  service { 'incus':
+    ensure     => running,
+    enable     => true,
+    hasstatus  => true,
+    hasrestart => true,
+  }
+
+  file_line { 'subuid_root':
+    ensure => present,
+    path   => '/etc/subuid',
+    line   => 'root:1000000:1000000000',
+    match  => '^root:',
+    notify => Service['incus'],
+  }
+
+  file_line { 'subgid_root':
+    ensure => present,
+    path   => '/etc/subgid',
+    line   => 'root:1000000:1000000000',
+    match  => '^root:',
+    notify => Service['incus'],
+  }
+}
diff --git a/modules/incus/templates/join_preseed.yaml.erb b/modules/incus/templates/join_preseed.yaml.erb
new file mode 100644
index 0000000..9491b08
--- /dev/null
+++ b/modules/incus/templates/join_preseed.yaml.erb
@@ -0,0 +1,20 @@
+config: {}
+networks: []
+storage_pools: []
+profiles: []
+projects: []
+cluster:
+  server_name: <%= @server_fqdn %>
+  enabled: true
+  member_config:
+  - entity: storage-pool
+    name: local
+    key: source
+    value: ""
+    description: '"source" property for storage pool "local"'
+  cluster_address: <%= @cluster_address %>:<%= @server_port %>
+  cluster_certificate: |
+    <%= @certificate %>
+  server_address: <%= @server_fqdn %>:<%= @server_port %>
+  cluster_token: <%= @cluster_token %>
+  cluster_certificate_path: ""

From e3e8b3484d18aeefc23c0373d133b2126d34c15a Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 01:20:59 +1100
Subject: [PATCH 19/89] chore: enable extra groups (#231)

- enable adding extra groups to the sysadmin user

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/231
---
 site/profiles/manifests/accounts/sysadmin.pp | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/site/profiles/manifests/accounts/sysadmin.pp b/site/profiles/manifests/accounts/sysadmin.pp
index f766f31..0c9050d 100644
--- a/site/profiles/manifests/accounts/sysadmin.pp
+++ b/site/profiles/manifests/accounts/sysadmin.pp
@@ -2,12 +2,22 @@
 class profiles::accounts::sysadmin(
   String $password,
   Array[String] $sshkeys = [],
+  Array[String] $extra_groups = [],
 ){
+
+  $default_groups = [
+    'adm',
+    'admins',
+    'systemd-journal'
+  ]
+
+  $groups = $extra_groups + $default_groups
+
   profiles::base::account {'sysadmin':
     username   => 'sysadmin',
     uid        => 1000,
     gid        => 1000,
-    groups     => ['adm', 'admins', 'systemd-journal'],
+    groups     => $groups,
     sshkeys    => $sshkeys,
     sudo_rules => ['sysadmin ALL=(ALL) NOPASSWD:ALL'],
     password   => $password,

From e025928d77662e8f6c5d70c45e8c04c6564c4734 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 01:53:25 +1100
Subject: [PATCH 20/89] chore: set secretid for puppetboard (#232)

- manage the secret_key for puppetboard
- required since module upgrade

https://github.com/voxpupuli/puppetboard/issues/721

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/232
---
 hieradata/roles/infra/puppetboard/server.eyaml | 1 +
 site/profiles/manifests/puppet/puppetboard.pp  | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 hieradata/roles/infra/puppetboard/server.eyaml

diff --git a/hieradata/roles/infra/puppetboard/server.eyaml b/hieradata/roles/infra/puppetboard/server.eyaml
new file mode 100644
index 0000000..29c7cb3
--- /dev/null
+++ b/hieradata/roles/infra/puppetboard/server.eyaml
@@ -0,0 +1 @@
+profiles::puppet::puppetboard::secret_key: ENC[PKCS7,MIIB+wYJKoZIhvcNAQcDoIIB7DCCAegCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAoCr9JTtrnMaVjJlEpA0NYNt/QAgTGiEmS3kPn2oJ80Ay7fUl79pop95uLQIfw6C3e8WGUhxYt31wmzent4Bfbced7uF/tjhP2cniKPmZGuf/tmVrGIHIh4T4g0Wz2K0DIU/dGWUlgE03ICRxXlAy0atjUuAp02ehj7bzLuMf+vZbghm8vnOoDCTKjeZNAJEkDvBkIzwUu9JiM9Gn6BzoLLGdEERqs/LR832vjjNmF7KUEH/Ntt47wbLdfCzawsZsBNLggEb1HNGu0aSb0qBXYBPRq6w1L/RU8gX2dCqGRbhDZymihebaOcwU1AtbLEBdHTYQpga+c4bcTz9rJplLtjCBvQYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQAdSjLEaZPK47TY7LDCB7b4CBkOTvHbi4nb+pB/Kng+Z3N9ocZPZdGbV6WzMNGxsxmhWXoi+XKEzEvfB10UhQwDcvZXd2W9hhiRkakq7+S0uLWzCX1jNMH2LXZAzoiyGM4FFvGpx7Z64OhmVrOJuKnxD+3zK6PZqvMb3g7CjZeAr5vyhcT/zO75krhJuYp11ZFpLCRcuASf0ru+Jq5OKD4+ZI/g==]
diff --git a/site/profiles/manifests/puppet/puppetboard.pp b/site/profiles/manifests/puppet/puppetboard.pp
index 08b49aa..c141e73 100644
--- a/site/profiles/manifests/puppet/puppetboard.pp
+++ b/site/profiles/manifests/puppet/puppetboard.pp
@@ -21,7 +21,7 @@ class profiles::puppet::puppetboard (
   Stdlib::Port $nginx_port      = 80,
   Stdlib::Host $nginx_vhost     = 'puppetboard.main.unkin.net',
   Array[Stdlib::Host]  $nginx_aliases = [],
-  #String[1] $secret_key         = "${fqdn_rand_string(32)}",
+  String[1] $secret_key         = "${fqdn_rand_string(32)}",
 ) {
 
   # store puppet-agents ssl settings/certname
@@ -37,7 +37,7 @@ class profiles::puppet::puppetboard (
     basedir             => $basedir,
     virtualenv_dir      => $virtualenv_dir,
     settings_file       => $settings_file,
-    #secret_key         => $secret_key,
+    secret_key          => $secret_key,
     default_environment => $default_environment,
     puppetdb_host       => $puppetdb_host,
     puppetdb_port       => 8081,

From e3046563a20a3dc666e4ee2873b6ad2f13cf0f85 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 02:04:13 +1100
Subject: [PATCH 21/89] chore: install consul from package (#233)

- upgrade to puppet-consul changed default install method to archive
- ensure package method is used
- dont manage the repo, consul is packaged by rpmbuilder

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/233
---
 hieradata/common.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index ece85d6..d2871a5 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -165,6 +165,9 @@ profiles::ntp::client::peers:
 profiles::base::puppet_servers:
   - 'prodinf01n01.main.unkin.net'
 
+consul::install_method: 'package'
+consul::manage_repo: false
+
 profiles::dns::master::basedir: '/var/named/sources'
 profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
 profiles::dns::base::use_ns: 'region'

From 06b458cb0e8a2f4dd9639bbb8534b477aeec5381 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 12:31:09 +1100
Subject: [PATCH 22/89] feat: reposync for almalinux 9.4 (in vault) (#234)

- sync baseos, ha, appstream and crb repos

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/234
---
 hieradata/roles/infra/reposync/syncer.yaml | 35 ++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index 2c1b63d..5d5c1ab 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -38,6 +38,41 @@ profiles::consul::client::node_rules:
 profiles::reposync::webserver::nginx_listen_mode: both
 profiles::reposync::webserver::nginx_cert_type: vault
 profiles::reposync::repos_list:
+  almalinux_9_4_baseos:
+    repository: 'baseos'
+    description: 'AlmaLinux 9.4 BaseOS'
+    osname: 'almalinux'
+    release: '9.4'
+    baseurl: 'https://vault.almalinux.org/9.4/BaseOS/x86_64/os/'
+    gpgkey: 'https://vault.almalinux.org/9.4/BaseOS/x86_64/os/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_4_appstream:
+    repository: 'appstream'
+    description: 'AlmaLinux 9.4 AppStream'
+    osname: 'almalinux'
+    release: '9.4'
+    baseurl: 'https://vault.almalinux.org/9.4/AppStream/x86_64/os/'
+    gpgkey: 'https://vault.almalinux.org/9.4/AppStream/x86_64/os/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_4_crb:
+    repository: 'crb'
+    description: 'AlmaLinux 9.4 CRB'
+    osname: 'almalinux'
+    release: '9.4'
+    baseurl: 'https://vault.almalinux.org/9.4/CRB/x86_64/os/'
+    gpgkey: 'https://vault.almalinux.org/9.4/CRB/x86_64/os/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_4_ha:
+    repository: 'ha'
+    description: 'AlmaLinux 9.4 HighAvailability'
+    osname: 'almalinux'
+    release: '9.4'
+    baseurl: 'https://vault.almalinux.org/9.4/HighAvailability/x86_64/os/'
+    gpgkey: 'https://vault.almalinux.org/9.4/HighAvailability/x86_64/os/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_4_extras:
+    repository: 'extras'
+    description: 'AlmaLinux 9.4 extras'
+    osname: 'almalinux'
+    release: '9.4'
+    baseurl: 'https://vault.almalinux.org/9.4/extras/x86_64/os/'
+    gpgkey: 'https://vault.almalinux.org/9.4/extras/x86_64/os/RPM-GPG-KEY-AlmaLinux-9'
   docker_stable_el8:
     repository: 'stable'
     description: 'Docker CE Stable EL8'

From d39d25d3f16babc9aa27b260589079b879678bc9 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 16:24:55 +1100
Subject: [PATCH 23/89] feat: add almalinux 9.5 repos using mirrorlist (#235)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/235
---
 hieradata/roles/infra/reposync/syncer.yaml | 35 ++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index 5d5c1ab..2ccd0ae 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -38,6 +38,41 @@ profiles::consul::client::node_rules:
 profiles::reposync::webserver::nginx_listen_mode: both
 profiles::reposync::webserver::nginx_cert_type: vault
 profiles::reposync::repos_list:
+  almalinux_9_5_baseos:
+    repository: 'baseos'
+    description: 'AlmaLinux 9.5 BaseOS'
+    osname: 'almalinux'
+    release: '9.5'
+    mirrorlist: 'https://mirrors.almalinux.org/mirrorlist/9.5/baseos'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_5_appstream:
+    repository: 'appstream'
+    description: 'AlmaLinux 9.5 AppStream'
+    osname: 'almalinux'
+    release: '9.5'
+    mirrorlist: 'https://mirrors.almalinux.org/mirrorlist/9.5/appstream'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_5_crb:
+    repository: 'crb'
+    description: 'AlmaLinux 9.5 CRB'
+    osname: 'almalinux'
+    release: '9.5'
+    mirrorlist: 'https://mirrors.almalinux.org/mirrorlist/9.5/crb'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_5_ha:
+    repository: 'ha'
+    description: 'AlmaLinux 9.5 HighAvailability'
+    osname: 'almalinux'
+    release: '9.5'
+    mirrorlist: 'https://mirrors.almalinux.org/mirrorlist/9.5/highavailability'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux-9'
+  almalinux_9_5_extras:
+    repository: 'extras'
+    description: 'AlmaLinux 9.5 extras'
+    osname: 'almalinux'
+    release: '9.5'
+    mirrorlist: 'https://mirrors.almalinux.org/mirrorlist/9.5/extras'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux-9'
   almalinux_9_4_baseos:
     repository: 'baseos'
     description: 'AlmaLinux 9.4 BaseOS'

From 45b061a0536d44c978265918d698a58bd675de07 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 17:05:03 +1100
Subject: [PATCH 24/89] feat: change almalinux9 to use packagerepo (#236)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/236
---
 hieradata/os/AlmaLinux/AlmaLinux9.yaml | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/hieradata/os/AlmaLinux/AlmaLinux9.yaml b/hieradata/os/AlmaLinux/AlmaLinux9.yaml
index 7c98e9c..f3f218e 100644
--- a/hieradata/os/AlmaLinux/AlmaLinux9.yaml
+++ b/hieradata/os/AlmaLinux/AlmaLinux9.yaml
@@ -3,12 +3,28 @@
 crypto_policies::policy: 'DEFAULT:SHA1'
 
 profiles::yum::global::repos:
+  baseos:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/baseos-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/baseos-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  extras:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/extras-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/extras-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  appstream:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/appstream-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/appstream-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
+  highavailability:
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/ha-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/ha-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
+    mirrorlist: absent
   crb:
     name: crb
     descr: crb repository
     target: /etc/yum.repos.d/crb.repo
-    baseurl: https://edgecache.query.consul/almalinux/%{facts.os.release.full}/CRB/%{facts.os.architecture}/os
-    gpgkey: http://edgecache.query.consul/almalinux/RPM-GPG-KEY-AlmaLinux-%{facts.os.release.major}
+    baseurl: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/crb-daily/%{facts.os.architecture}/os/
+    gpgkey: https://packagerepo.service.consul/almalinux/%{facts.os.release.full}/crb-daily/%{facts.os.architecture}/os//RPM-GPG-KEY-AlmaLinux-9
     mirrorlist: absent
   unkin:
     name: unkin

From 427fe352b48d73a32ec18070af1895e63a773050 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 17:13:54 +1100
Subject: [PATCH 25/89] feat: debian package for consul not managed (#237)

- change debian hosts to use the url method to download consul

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/237
---
 hieradata/os/Debian/all_releases.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hieradata/os/Debian/all_releases.yaml b/hieradata/os/Debian/all_releases.yaml
index bd8f426..0caa1b1 100644
--- a/hieradata/os/Debian/all_releases.yaml
+++ b/hieradata/os/Debian/all_releases.yaml
@@ -13,3 +13,6 @@ profiles::packages::include:
 
 lm-sensors::package: lm-sensors
 networking::nwmgr_dns_none: false
+
+consul::install_method: 'url'
+consul::manage_repo: false

From 6cb249ffbceca7f1820124806f078d4b78562faa Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 17:51:33 +1100
Subject: [PATCH 26/89] fix: backtrack to 9.2.0 for postgresql (#238)

- no parameter named 'instance'
- no parameter named 'port'

downgrading due to incompatibilities between the latest version of puppetdb and postgresql

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/238
---
 Puppetfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Puppetfile b/Puppetfile
index d38a073..b559a1b 100644
--- a/Puppetfile
+++ b/Puppetfile
@@ -9,8 +9,8 @@ mod 'puppetlabs-vcsrepo', '7.0.0'
 mod 'puppetlabs-yumrepo_core', '2.1.0'
 mod 'puppetlabs-apt', '10.0.1'
 mod 'puppetlabs-lvm', '3.0.1'
-mod 'puppetlabs-puppetdb', '8.1.0'
-mod 'puppetlabs-postgresql', '10.5.0'
+mod 'puppetlabs-puppetdb', '7.14.0'
+mod 'puppetlabs-postgresql', '9.2.0'
 mod 'puppetlabs-firewall', '8.1.4'
 mod 'puppetlabs-accounts', '8.2.2'
 mod 'puppetlabs-mysql', '16.2.0'

From 829b1b05fd67a9894817fb82975054aa7e9c2c3f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 30 Mar 2025 18:40:09 +1100
Subject: [PATCH 27/89] feat: cleanup consul from url install (#239)

- set bind_dir to be /usr/bin for rhel, /usr/local/bin for debian
- remove url-installed consul from rhel

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/239
---
 hieradata/common.yaml                    | 1 +
 hieradata/os/Debian/all_releases.yaml    | 1 +
 site/profiles/manifests/consul/client.pp | 6 ++++++
 3 files changed, 8 insertions(+)

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index d2871a5..ecd78e5 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -167,6 +167,7 @@ profiles::base::puppet_servers:
 
 consul::install_method: 'package'
 consul::manage_repo: false
+consul::bin_dir: /usr/bin
 
 profiles::dns::master::basedir: '/var/named/sources'
 profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
diff --git a/hieradata/os/Debian/all_releases.yaml b/hieradata/os/Debian/all_releases.yaml
index 0caa1b1..efd71f9 100644
--- a/hieradata/os/Debian/all_releases.yaml
+++ b/hieradata/os/Debian/all_releases.yaml
@@ -16,3 +16,4 @@ networking::nwmgr_dns_none: false
 
 consul::install_method: 'url'
 consul::manage_repo: false
+consul::bin_dir: /usr/local/bin
diff --git a/site/profiles/manifests/consul/client.pp b/site/profiles/manifests/consul/client.pp
index d1d82d8..fa229c8 100644
--- a/site/profiles/manifests/consul/client.pp
+++ b/site/profiles/manifests/consul/client.pp
@@ -85,4 +85,10 @@ class profiles::consul::client (
     require => File['/root/.config'],
   }
 
+  # cleanup /usr/local/bin/consul which was created by url install method
+  if $facts['os']['family'] == 'RedHat' {
+    file {'/usr/local/bin/consul':
+      ensure  => absent,
+    }
+  }
 }

From 978013f325ff6a3626ff8d78240dd7f560972d8f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 31 Mar 2025 22:49:47 +1100
Subject: [PATCH 28/89] chore: set default nameservers (#240)

- if no nameservers are returned from puppetdb query, use default

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/240
---
 site/profiles/manifests/dns/base.pp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp
index e22e964..12d2d99 100644
--- a/site/profiles/manifests/dns/base.pp
+++ b/site/profiles/manifests/dns/base.pp
@@ -2,7 +2,7 @@
 class profiles::dns::base (
   String $ns_role = undef,
   Array  $search  = [],
-  Array  $nameservers = ['8.8.8.8', '1.1.1.1'],
+  Array  $nameservers = ['198.18.13.12', '198.18.13.13'],
   Enum[
     'all',
     'region',
@@ -23,6 +23,12 @@ class profiles::dns::base (
     }
   }
 
+  # if nameservers not returned from puppetdb, use default
+  $use_nameservers = empty($nameserver_array) ? {
+    true  => $nameservers,
+    false => $nameserver_array,
+  }
+
   # if search is undef, fallback to domainname from facts
   if $search == [] {
     $search_array = [$::facts['networking']['domain']]
@@ -32,7 +38,7 @@ class profiles::dns::base (
 
   # include resolvconf class
   class { 'profiles::dns::resolvconf':
-    nameservers    => sort($nameserver_array),
+    nameservers    => sort($use_nameservers),
     search_domains => sort($search_array),
   }
 

From 95bc2716cfb35c9f1eb8044d9e9357e8865b0ce1 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 31 Mar 2025 23:14:05 +1100
Subject: [PATCH 29/89] neoloc/incus_deploy (#241)

feat: deploy incus

- manage sysctl based on incus recommendations
- manage limits based on incus recommendations
- manage zpools and zfs datasets
- add incus hiera settings

feat: manage repo for zfs

- dont use zfs module to manage repo, use profiles::yum::global::repos

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/241
---
 hieradata/roles/infra/incus/node.yaml | 81 +++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index a0c8ecb..3ec69b9 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -1,4 +1,8 @@
 ---
+hiera_include:
+  - incus
+  - zfs
+
 profiles::pki::vault::alt_names:
   - incus.service.consul
   - incus.query.consul
@@ -31,3 +35,80 @@ profiles::consul::client::node_rules:
   - resource: service
     segment: incus
     disposition: write
+
+# additional repos
+profiles::yum::global::repos:
+  baseos:
+    name: zfs-kmod
+    descr: zfs-kmod repository
+    target: /etc/yum.repos.d/zfs-kmod.repo
+    baseurl: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-openzfs-2022
+    mirrorlist: absent
+
+
+# zfs settings
+zfs::manage_repo: false
+zfs::zfs_arc_min: ~
+zfs::zfs_arc_max: 4294967296 # 4GB
+zfs::zpools:
+  fastpool:
+    ensure: present
+    disk: /dev/nvme1n1
+    ashift: 12
+zfs::datasets:
+  fastpool:
+    canmount: 'off'
+    acltype: posix
+    atime: 'off'
+    relatime: 'off'
+    compression: 'zstd'
+    xattr: 'sa'
+  fastpool/data:
+    canmount: 'on'
+    mountpoint: '/data'
+
+# manage incus
+incus::cluster::members_lookup: true
+incus::cluster::members_role: roles::infra::incus::node
+incus::cluster::master: prodnxsr0009
+
+# add sysadmin to incus-admin group
+profiles::accounts::sysadmin::extra_groups:
+  - incus-admin
+
+# sysctl recommendations
+sysctl::base::values:
+  fs.aio-max-nr:
+    value: '524288'
+  fs.inotify.max_queued_events:
+    value: '1048576'
+  fs.inotify.max_user_instances:
+    value: '1048576'
+  fs.inotify.max_user_watches:
+    value: '1048576'
+  kernel.dmesg_restrict:
+    value: '1'
+  kernel.keys.maxbytes:
+    value: '2000000'
+  kernel.keys.maxkeys:
+    value: '2000'
+  net.core.bpf_jit_limit:
+    value: '1000000000'
+  net.ipv4.neigh.default.gc_thresh3:
+    value: '8192'
+  net.ipv6.neigh.default.gc_thresh3:
+    value: '8192'
+  vm.max_map_count:
+    value: '262144'
+
+# limits.d recommendations
+limits::entries:
+  '*/nofile':
+    both: 1048576
+  'root/nofile':
+    both: 1048576
+  '*/memlock':
+    both: unlimited
+  'root/memlock':
+    both: unlimited

From d87983d8fc7146288983f1bf74d32d2a0fa1ad14 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 2 Apr 2025 20:27:11 +1100
Subject: [PATCH 30/89] chore: add sysadmin user after first run (#242)

- enables extra_groups to function correctly

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/242
---
 site/profiles/manifests/firstrun/init.pp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/site/profiles/manifests/firstrun/init.pp b/site/profiles/manifests/firstrun/init.pp
index cdb70e7..ce39cc1 100644
--- a/site/profiles/manifests/firstrun/init.pp
+++ b/site/profiles/manifests/firstrun/init.pp
@@ -11,9 +11,8 @@ class profiles::firstrun::init {
   # set the motd
   include profiles::base::motd
 
-  # create the sysadmin account
+  # create groups
   include profiles::base::groups
-  include profiles::accounts::sysadmin
 
   # mark the firstrun as done
   include profiles::firstrun::complete

From 9dc88e6db679d63843b14da82544ae5213ddfed5 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 2 Apr 2025 20:35:04 +1100
Subject: [PATCH 31/89] feat: deep merge zpools/datasets (#243)

- change prodnxsr0009 to use nvme0n1 as zfs device

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/243
---
 hieradata/common.yaml                            | 6 ++++++
 hieradata/nodes/prodnxsr0009.main.unkin.net.yaml | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index ecd78e5..89cec33 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -143,6 +143,12 @@ lookup_options:
   limits::entries:
     merge:
       strategy: deep
+  zfs::zpools:
+    merge:
+      strategy: deep
+  zfs::datasets:
+    merge:
+      strategy: deep
 
 facts_path: '/opt/puppetlabs/facter/facts.d'
 
diff --git a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
index 6a43056..8ae6946 100644
--- a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
@@ -8,3 +8,8 @@ networking::interfaces:
 networking::routes:
   default:
     gateway: 198.18.15.254
+
+zfs::zpools:
+  fastpool:
+    ensure: present
+    disk: /dev/nvme0n1

From 06666fe488837ed5484c6d887f6cfdc2e99e7203 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 2 Apr 2025 21:02:08 +1100
Subject: [PATCH 32/89] fix: resolve issue with baseos in el9 (#244)

- was not correctly provisioning the baseos repo for el9 incus hosts

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/244
---
 hieradata/roles/infra/incus/node.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 3ec69b9..229a047 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -38,7 +38,7 @@ profiles::consul::client::node_rules:
 
 # additional repos
 profiles::yum::global::repos:
-  baseos:
+  zfs-kmod:
     name: zfs-kmod
     descr: zfs-kmod repository
     target: /etc/yum.repos.d/zfs-kmod.repo

From c225564bdb408e7360f500703270809f1123c2ba Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 6 Apr 2025 16:38:04 +1000
Subject: [PATCH 33/89] feat: continue incus implementation (#245)

- migrate to systemd-networkd
- setup dummy, bridge and static/ethernet interfaces
- manage sshd.service droping to start ssh after networking is online
- enable ip forewarding
- add fastpool/data/incus dataset
- enable ospf and frr
- add loopback0 as ssh listenaddress
- add loopback1/2 for ceph cluster/public traffic

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/245
---
 .../nodes/prodnxsr0009.main.unkin.net.yaml    | 21 +++---
 .../nodes/prodnxsr0010.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0011.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0012.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0013.main.unkin.net.yaml    | 13 ++--
 hieradata/roles/infra/incus/node.yaml         | 70 ++++++++++++++++++
 modules/libs/lib/facter/subnet_facts.rb       | 13 +++-
 modules/networking/manifests/bridge.pp        | 22 ++++++
 modules/networking/manifests/dummy.pp         | 18 +++++
 modules/networking/manifests/init.pp          | 72 +++++++++++++------
 modules/networking/manifests/static.pp        | 26 +++++++
 .../networking/templates/bridge.netdev.erb    |  3 +
 modules/networking/templates/dummy.netdev.erb |  3 +
 .../networking/templates/networkd-link.erb    |  8 +++
 .../networking/templates/networkd-network.erb | 37 ++++++++++
 site/profiles/manifests/base.pp               |  1 +
 site/profiles/manifests/ssh/service.pp        | 15 ++++
 17 files changed, 310 insertions(+), 51 deletions(-)
 create mode 100644 modules/networking/manifests/bridge.pp
 create mode 100644 modules/networking/manifests/dummy.pp
 create mode 100644 modules/networking/manifests/static.pp
 create mode 100644 modules/networking/templates/bridge.netdev.erb
 create mode 100644 modules/networking/templates/dummy.netdev.erb
 create mode 100644 modules/networking/templates/networkd-link.erb
 create mode 100644 modules/networking/templates/networkd-network.erb
 create mode 100644 site/profiles/manifests/ssh/service.pp

diff --git a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
index 8ae6946..a1cc562 100644
--- a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
@@ -1,15 +1,18 @@
 ---
+networking_loopback0_ip: 198.18.19.9  # management loopback
+networking_loopback1_ip: 198.18.22.9  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.9  # ceph-public loopback
+networking_br10_ip: 198.18.25.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:8d
     ipaddress: 198.18.15.9
-  enp3s0:
-    ipaddress: 10.18.15.9
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:5d
+    ipaddress: 198.18.21.9
 
-zfs::zpools:
-  fastpool:
-    ensure: present
-    disk: /dev/nvme0n1
+#zfs::zpools:
+#  fastpool:
+#    ensure: present
+#    disk: /dev/nvme0n1
diff --git a/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
index d9e7592..a7e0f91 100644
--- a/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.10  # management loopback
+networking_loopback1_ip: 198.18.22.10  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.10  # ceph-public loopback
+networking_br10_ip: 198.18.26.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:37
     ipaddress: 198.18.15.10
-  enp3s0:
-    ipaddress: 10.18.15.10
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:de
+    ipaddress: 198.18.21.10
diff --git a/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
index 28a45ab..e146b42 100644
--- a/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.11  # management loopback
+networking_loopback1_ip: 198.18.22.11  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.11  # ceph-public loopback
+networking_br10_ip: 198.18.27.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:0f
     ipaddress: 198.18.15.11
-  enp3s0:
-    ipaddress: 10.18.15.11
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:55
+    ipaddress: 198.18.21.11
diff --git a/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
index fbb68c4..c309a59 100644
--- a/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.12  # management loopback
+networking_loopback1_ip: 198.18.22.12  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.12  # ceph-public loopback
+networking_br10_ip: 198.18.28.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:4f:05:1e
     ipaddress: 198.18.15.12
-  enp3s0:
-    ipaddress: 10.18.15.12
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:e5
+    ipaddress: 198.18.21.12
diff --git a/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
index 221e494..86221c3 100644
--- a/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.13  # management loopback
+networking_loopback1_ip: 198.18.22.13  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.13  # ceph-public loopback
+networking_br10_ip: 198.18.29.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:4f:04:b0
     ipaddress: 198.18.15.13
-  enp3s0:
-    ipaddress: 10.18.15.13
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:36
+    ipaddress: 198.18.21.13
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 229a047..720285e 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -1,8 +1,12 @@
 ---
 hiera_include:
+  - frrouting
   - incus
   - zfs
 
+profiles::packages::include:
+  bridge-utils: {}
+
 profiles::pki::vault::alt_names:
   - incus.service.consul
   - incus.query.consul
@@ -46,6 +50,65 @@ profiles::yum::global::repos:
     gpgkey: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-openzfs-2022
     mirrorlist: absent
 
+# networking
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+#networking::use_networkd: true
+networking::interfaces:
+  enp2s0:
+    type: physical
+    txqueuelen: 10000
+    forwarding: true
+  enp3s0:
+    type: physical
+    mtu: 9000
+    txqueuelen: 10000
+    forwarding: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  loopback1:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback1_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  loopback2:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback2_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  br10:
+    type: bridge
+    bridge: true
+    ipaddress: "%{hiera('networking_br10_ip')}"
+    netmask: 255.255.255.0
+    nocarrier: true
+    forwarding: true
+
+# frrouting
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  enp2s0:
+    area: 0.0.0.0
+  enp3s0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+  loopback1:
+    area: 0.0.0.0
+  loopback2:
+    area: 0.0.0.0
+  br10:
+    area: 0.0.0.0
+
+# add loopback interfaces to ssh list
+ssh::server::options:
+  ListenAddress:
+    - "%{hiera('networking_loopback0_ip')}"
 
 # zfs settings
 zfs::manage_repo: false
@@ -67,6 +130,9 @@ zfs::datasets:
   fastpool/data:
     canmount: 'on'
     mountpoint: '/data'
+  fastpool/data/incus:
+    canmount: 'on'
+    mountpoint: '/data/incus'
 
 # manage incus
 incus::cluster::members_lookup: true
@@ -101,6 +167,10 @@ sysctl::base::values:
     value: '8192'
   vm.max_map_count:
     value: '262144'
+  net.ipv4.conf.all.forwarding:
+    value: '1'
+  net.ipv6.conf.all.forwarding:
+    value: '1'
 
 # limits.d recommendations
 limits::entries:
diff --git a/modules/libs/lib/facter/subnet_facts.rb b/modules/libs/lib/facter/subnet_facts.rb
index 458c8e0..28325fd 100644
--- a/modules/libs/lib/facter/subnet_facts.rb
+++ b/modules/libs/lib/facter/subnet_facts.rb
@@ -10,7 +10,18 @@ class SubnetAttributes
     '198.18.15.0/24' => { environment: 'prod', region: 'syd1', country: 'au' },
     '198.18.16.0/24' => { environment: 'test', region: 'syd1', country: 'au' },
     '198.18.17.0/24' => { environment: 'prod', region: 'drw1', country: 'au' },
-    '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' }
+    '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' },
+    '198.18.19.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # loopbacks
+    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # RESERVED
+    '198.18.21.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # physical network 2.5gbe
+    '198.18.22.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph cluster
+    '198.18.23.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph public
+    '198.18.24.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # dmz 1
+    '198.18.25.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0009
+    '198.18.26.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0010
+    '198.18.27.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0011
+    '198.18.28.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0012
+    '198.18.29.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }  # common node0013
   }.freeze
 
   # Default attributes if no subnet matches, also defined as a constant
diff --git a/modules/networking/manifests/bridge.pp b/modules/networking/manifests/bridge.pp
new file mode 100644
index 0000000..7fd30f1
--- /dev/null
+++ b/modules/networking/manifests/bridge.pp
@@ -0,0 +1,22 @@
+# manage bridges and bridge slaves
+define networking::bridge (
+  String                $type,
+  Optional[Stdlib::IP::Address] $ipaddress,
+  Optional[Stdlib::IP::Address] $netmask   = undef,
+  Optional[Stdlib::IP::Address] $gateway   = undef,
+  Optional[Boolean]             $nocarrier = undef,
+  Boolean               $bridge = true,
+  Integer[100-9200]     $mtu    = 1500,
+  Optional[Boolean]     $forwarding = false,
+) {
+  include systemd
+
+  systemd::network { "${title}.netdev":
+    content => template('networking/bridge.netdev.erb'),
+  }
+
+  # Use shared template, it will detect bridge=true and skip Address/DNS/etc
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+}
diff --git a/modules/networking/manifests/dummy.pp b/modules/networking/manifests/dummy.pp
new file mode 100644
index 0000000..cae8e10
--- /dev/null
+++ b/modules/networking/manifests/dummy.pp
@@ -0,0 +1,18 @@
+# manage dummy/loopback interfaces
+define networking::dummy (
+  String                $type,
+  Stdlib::IP::Address   $ipaddress,
+  Stdlib::IP::Address   $netmask,
+  Integer[100-9200]     $mtu = 1500,
+  Optional[Boolean]     $forwarding = false,
+) {
+  include systemd
+
+  systemd::network { "${title}.netdev":
+    content => template('networking/dummy.netdev.erb'),
+  }
+
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+}
diff --git a/modules/networking/manifests/init.pp b/modules/networking/manifests/init.pp
index d2076ca..8a28690 100644
--- a/modules/networking/manifests/init.pp
+++ b/modules/networking/manifests/init.pp
@@ -4,37 +4,67 @@ class networking (
   Hash $interface_defaults = {},
   Hash $routes = {},
   Hash $route_defaults = {},
+  Boolean $use_networkd = lookup('systemd::manage_networkd'),
 ){
 
   include network
   include networking::params
 
-  # manage interfaces
-  $interfaces.each | $interface, $data | {
-    $merged_data = merge($interface_defaults, $data)
-    network_config { $interface:
-      *      => $merged_data,
-      notify => Exec['networking_reload_network'],
-    }
-  }
+  if $use_networkd {
 
-  # manage routes
-  $routes.each | $route, $data | {
-    $merged_data = merge($route_defaults, $data)
-    network_route { $route:
-      *      => $merged_data,
-      notify => Exec['networking_reload_network'],
+    include systemd
+
+    service { 'NetworkManager':
+      ensure => 'stopped',
+      enable => false,
+    }
+
+    $interfaces.each |String $iface, Hash $data| {
+      $type   = $data['type']
+      #$params = $data.filter |$key, $value| { $key != 'type' }
+
+      case $type {
+        'bridge': { networking::bridge { $iface: * => $data } }
+        'dummy': { networking::dummy { $iface: * => $data } }
+        'static': { networking::static { $iface: * => $data } }
+        'physical': { networking::static { $iface: * => $data } }
+        default: {
+          fail("Unsupported interface type '${type}' for interface '${iface}'")
+        }
+      }
+    }
+  }else{
+    # manage interfaces
+    $interfaces.each | $interface, $data | {
+      $merged_data = merge($interface_defaults, $data)
+      network_config { $interface:
+        *      => $merged_data,
+        notify => Exec['networking_reload_network'],
+      }
+    }
+
+    # manage routes
+    $routes.each | $route, $data | {
+      $merged_data = merge($route_defaults, $data)
+      network_route { $route:
+        *      => $merged_data,
+        notify => Exec['networking_reload_network'],
+      }
     }
   }
 
   # determine which networking service to restart
-  $restart_command = $facts['os']['family'] ? {
-    'RedHat' => $facts['os']['release']['major'] ? {
-      '8' => '/usr/bin/systemctl restart network',
-      '9' => '/usr/bin/systemctl restart NetworkManager',
-    },
-    'Debian' => '/usr/bin/systemctl restart networking',
-    default  => fail('Unsupported OS in networking-restart-command'),
+  $restart_command = $use_networkd ? {
+    true    => '/usr/bin/systemctl restart systemd-networkd',
+    default => $facts['os']['family'] ? {
+      'RedHat' => $facts['os']['release']['major'] ? {
+        '8' => '/usr/bin/systemctl restart network',
+        '9' => '/usr/bin/systemctl restart NetworkManager',
+        default => fail('Unsupported RedHat OS release for networking restart'),
+      },
+      'Debian' => '/usr/bin/systemctl restart networking',
+      default  => fail('Unsupported OS in networking-restart-command'),
+    }
   }
 
   # restart network/networking only if $restart_networking boolean is true
diff --git a/modules/networking/manifests/static.pp b/modules/networking/manifests/static.pp
new file mode 100644
index 0000000..018ef89
--- /dev/null
+++ b/modules/networking/manifests/static.pp
@@ -0,0 +1,26 @@
+# manage static interfaces
+define networking::static (
+  String                $type,
+  Stdlib::IP::Address   $ipaddress,
+  Stdlib::IP::Address   $netmask  = '255.255.255.0',
+  Integer[100-9200]     $mtu      = 1500,
+  Optional[Boolean]     $forwarding = false,
+  Optional[Stdlib::IP::Address]         $gateway = undef,
+  Optional[Array[Stdlib::IP::Address]]  $dns = undef,
+  Optional[Array[Stdlib::Fqdn]]         $domains = undef,
+  Optional[Integer[0-4096]]             $vlan = undef,
+  Optional[Variant[Boolean,String]]     $bridge = undef,
+  Optional[Integer[0-4294967294]]       $txqueuelen = undef,
+  Optional[Stdlib::MAC]                 $mac = undef,
+) {
+  include systemd
+
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+  #if $type == 'physical' and $mac {
+  #  systemd::network { "${title}.link":
+  #    content => template('networking/networkd-link.erb'),
+  #  }
+  #}
+}
diff --git a/modules/networking/templates/bridge.netdev.erb b/modules/networking/templates/bridge.netdev.erb
new file mode 100644
index 0000000..94253e0
--- /dev/null
+++ b/modules/networking/templates/bridge.netdev.erb
@@ -0,0 +1,3 @@
+[NetDev]
+Name=<%= @title %>
+Kind=bridge
diff --git a/modules/networking/templates/dummy.netdev.erb b/modules/networking/templates/dummy.netdev.erb
new file mode 100644
index 0000000..05ef8f4
--- /dev/null
+++ b/modules/networking/templates/dummy.netdev.erb
@@ -0,0 +1,3 @@
+[NetDev]
+Name=<%= @title %>
+Kind=dummy
diff --git a/modules/networking/templates/networkd-link.erb b/modules/networking/templates/networkd-link.erb
new file mode 100644
index 0000000..d45240d
--- /dev/null
+++ b/modules/networking/templates/networkd-link.erb
@@ -0,0 +1,8 @@
+[Match]
+MACAddress=<%= @mac %>
+
+[Link]
+MTUBytes=<%= @mtu %>
+<% if @txqueuelen and @txqueuelen >= 1 -%>
+TransmitQueueLength=<%= @txqueuelen %>
+<% end -%>
diff --git a/modules/networking/templates/networkd-network.erb b/modules/networking/templates/networkd-network.erb
new file mode 100644
index 0000000..298304d
--- /dev/null
+++ b/modules/networking/templates/networkd-network.erb
@@ -0,0 +1,37 @@
+[Match]
+Name=<%= @title %>
+
+[Network]
+<% if @ipaddress && @netmask -%>
+Address=<%= @ipaddress %>/<%= IPAddr.new(@netmask).to_i.to_s(2).count('1') %>
+<% end -%>
+<% if @gateway -%>
+Gateway=<%= @gateway %>
+<% end -%>
+<% if @dns -%>
+DNS=<%= Array(@dns).join(' ') %>
+<% end -%>
+<% if @domains -%>
+Domains=<%= Array(@domains).join(' ') %>
+<% end -%>
+<% if @bridge and @bridge != true -%>
+Bridge=<%= @bridge %>
+<% end -%>
+<% if @vlan -%>
+VLAN=<%= @vlan %>
+<% end -%>
+<% if @nocarrier and @nocarrier == true -%>
+ConfigureWithoutCarrier=true
+DuplicateAddressDetection=none
+RequiredForOnline=no-carrier
+<% end -%>
+<% if @type == 'dummy' -%>
+LinkLocalAddressing=no
+ActivationPolicy=always-up
+<% end -%>
+<% if @forwarding and @forwarding == true -%>
+IPForward=true
+<% end -%>
+
+[Link]
+MTUBytes=<%= @mtu %>
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index c1a4feb..4ec5150 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -33,6 +33,7 @@ class profiles::base (
     include profiles::pki::vault
     include profiles::ssh::sign
     include profiles::ssh::knownhosts
+    include profiles::ssh::service
     include profiles::cloudinit::init
     include profiles::metrics::default
     include profiles::helpers::node_lookup
diff --git a/site/profiles/manifests/ssh/service.pp b/site/profiles/manifests/ssh/service.pp
new file mode 100644
index 0000000..e334016
--- /dev/null
+++ b/site/profiles/manifests/ssh/service.pp
@@ -0,0 +1,15 @@
+# profiles::ssh::service
+# saz-ssh manages the service, this is just some additional stuff
+class profiles::ssh::service {
+
+  # set sshd to start
+  systemd::manage_dropin { 'after-network-online.conf':
+    ensure     => present,
+    unit       => 'sshd.service',
+    unit_entry => {
+      'After' => [
+        'network-online.target',
+      ],
+    },
+  }
+}

From b6ea353cfb905477029adf8e7301b9fbf8e25692 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 6 Apr 2025 16:44:16 +1000
Subject: [PATCH 34/89] feat: update dns resolver acls (#246)

- add dmz acl
- add common acl
- add loopback/ceph/physical subnets to main acl

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/246
---
 hieradata/roles/infra/dns/resolver.yaml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index f39588c..f978202 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -10,6 +10,22 @@ profiles::dns::resolver::acls:
       - 198.18.15.0/24
       - 198.18.16.0/24
       - 198.18.17.0/24
+      - 198.18.18.0/24
+      - 198.18.19.0/24
+      - 198.18.20.0/24
+      - 198.18.21.0/24
+      - 198.18.22.0/24
+      - 198.18.23.0/24
+  acl-dmz:
+    addresses:
+      - 198.18.24.0/24
+  acl-common:
+    addresses:
+      - 198.18.25.0/24
+      - 198.18.26.0/24
+      - 198.18.27.0/24
+      - 198.18.28.0/24
+      - 198.18.29.0/24
   acl-nomad-jobs:
     addresses:
       - 198.18.64.0/24
@@ -83,3 +99,4 @@ profiles::dns::resolver::views:
     match_clients:
       - acl-main.unkin.net
       - acl-nomad-jobs
+      - acl-common

From 83d0b31753a5f7caaeff9f924ca8d00ff896574a Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 6 Apr 2025 19:24:39 +1000
Subject: [PATCH 35/89] fix: set default for use_networkd (#247)

- resolving issue where the systemd::manage_networkd is missing for most
  hosts, setting a default

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/247
---
 modules/networking/manifests/init.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/networking/manifests/init.pp b/modules/networking/manifests/init.pp
index 8a28690..2cce6c7 100644
--- a/modules/networking/manifests/init.pp
+++ b/modules/networking/manifests/init.pp
@@ -4,7 +4,7 @@ class networking (
   Hash $interface_defaults = {},
   Hash $routes = {},
   Hash $route_defaults = {},
-  Boolean $use_networkd = lookup('systemd::manage_networkd'),
+  Boolean $use_networkd = lookup('systemd::manage_networkd', undef, undef, false),
 ){
 
   include network

From 0e3dd4d7d0c6e43feda764ae1650ed77194170ea Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 6 Apr 2025 23:56:50 +1000
Subject: [PATCH 36/89] feat: initialise barebones server (#248)

- manage incus servers init

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/248
---
 hieradata/roles/infra/incus/node.yaml         |  7 ++--
 modules/incus/lib/facter/incus.rb             |  2 +-
 modules/incus/manifests/init.pp               | 24 +++++++++++++-
 modules/incus/templates/join_preseed.yaml.erb | 32 +++++++++----------
 4 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 720285e..d250eb1 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -135,9 +135,10 @@ zfs::datasets:
     mountpoint: '/data/incus'
 
 # manage incus
-incus::cluster::members_lookup: true
-incus::cluster::members_role: roles::infra::incus::node
-incus::cluster::master: prodnxsr0009
+incus::init: true
+incus::bridge: br10
+incus::server_port: 8443
+incus::server_addr: "%{hiera('networking_loopback0_ip')}"
 
 # add sysadmin to incus-admin group
 profiles::accounts::sysadmin::extra_groups:
diff --git a/modules/incus/lib/facter/incus.rb b/modules/incus/lib/facter/incus.rb
index e9639f6..585bd28 100644
--- a/modules/incus/lib/facter/incus.rb
+++ b/modules/incus/lib/facter/incus.rb
@@ -13,6 +13,6 @@ Facter.add(:incus) do
     next {} if incus_output.empty? # Return an empty fact if there's no output
 
     # Parse the output as YAML and return it
-    YAML.safe_load(incus_output)
+    YAML.safe_load(incus_output, permitted_classes: [Symbol, Time, Date])
   end
 end
diff --git a/modules/incus/manifests/init.pp b/modules/incus/manifests/init.pp
index 0e7dc19..d681fb9 100644
--- a/modules/incus/manifests/init.pp
+++ b/modules/incus/manifests/init.pp
@@ -1,9 +1,14 @@
 class incus (
-  Array[String] $packages   = [
+  Array[String] $packages     = [
     'incus',
     'incus-tools',
     'incus-client'
   ],
+  Boolean       $cluster      = false,
+  Boolean       $init         = true,
+  String        $bridge       = 'incusbr0',
+  Stdlib::Port        $server_port = 8443,
+  Stdlib::IP::Address $server_addr = $facts['networking']['ip'],
 ) {
 
   package { $packages:
@@ -32,4 +37,21 @@ class incus (
     match  => '^root:',
     notify => Service['incus'],
   }
+
+  if $init {
+    file {'/root/incus.preseed.yaml':
+      ensure  => file,
+      owner   => root,
+      group   => root,
+      content => template('incus/join_preseed.yaml.erb')
+    }
+
+    exec { 'initiate_incus':
+      path        => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'],
+      command     => 'cat /root/incus.preseed.yaml | incus admin init --preseed && touch /root/.incus_initialized',
+      refreshonly => true,
+      creates     => '/root/.incus_initialized',
+      subscribe   => File['/root/incus.preseed.yaml'],
+    }
+  }
 }
diff --git a/modules/incus/templates/join_preseed.yaml.erb b/modules/incus/templates/join_preseed.yaml.erb
index 9491b08..71026af 100644
--- a/modules/incus/templates/join_preseed.yaml.erb
+++ b/modules/incus/templates/join_preseed.yaml.erb
@@ -1,20 +1,18 @@
-config: {}
+config:
+  core.https_address: <%= @server_fqdn %>:<%= @server_port %>
 networks: []
 storage_pools: []
-profiles: []
+storage_volumes: []
+profiles:
+- config: {}
+  description: ""
+  devices:
+    eth0:
+      name: eth0
+      nictype: bridged
+      parent: <%= @bridge %>
+      type: nic
+  name: default
+  project: default
 projects: []
-cluster:
-  server_name: <%= @server_fqdn %>
-  enabled: true
-  member_config:
-  - entity: storage-pool
-    name: local
-    key: source
-    value: ""
-    description: '"source" property for storage pool "local"'
-  cluster_address: <%= @cluster_address %>:<%= @server_port %>
-  cluster_certificate: |
-    <%= @certificate %>
-  server_address: <%= @server_fqdn %>:<%= @server_port %>
-  cluster_token: <%= @cluster_token %>
-  cluster_certificate_path: ""
+cluster: null

From 8c76e71dc4cb936acdf7d12894790040fc8f5d6d Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 7 Apr 2025 11:04:12 +1000
Subject: [PATCH 37/89] chore: set core.https_address for incus (#249)

- check the current config and update core.https_address if its wrong

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/249
---
 modules/incus/manifests/init.pp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/incus/manifests/init.pp b/modules/incus/manifests/init.pp
index d681fb9..6933997 100644
--- a/modules/incus/manifests/init.pp
+++ b/modules/incus/manifests/init.pp
@@ -54,4 +54,12 @@ class incus (
       subscribe   => File['/root/incus.preseed.yaml'],
     }
   }
+
+  # set core.https_address
+  if $facts['incus']['config']['core.https_address'] != "${server_addr}:${server_port}" {
+    exec { 'incus_config_set_core_https_address':
+      path    => ['/bin', '/usr/bin'],
+      command => "incus config set core.https_address ${server_addr}:${server_port}",
+    }
+  }
 }

From 25b06cde22c86056caa8909d1d152b0b69f60a79 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Tue, 15 Apr 2025 00:04:14 +1000
Subject: [PATCH 38/89] feat: move bridge management to incus (#250)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/250
---
 hieradata/roles/infra/incus/node.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index d250eb1..cb70475 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -79,13 +79,6 @@ networking::interfaces:
     ipaddress: "%{hiera('networking_loopback2_ip')}"
     netmask: 255.255.255.255
     mtu: 9000
-  br10:
-    type: bridge
-    bridge: true
-    ipaddress: "%{hiera('networking_br10_ip')}"
-    netmask: 255.255.255.0
-    nocarrier: true
-    forwarding: true
 
 # frrouting
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"

From 0fe44cf4e2c3d592ddae713b0f2bf1d02ca49d15 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Tue, 15 Apr 2025 02:21:55 +1000
Subject: [PATCH 39/89] feat: add frr repos (#251)

- add frr/stable/el8
- add frr/stable/el9
- add frr/extras/el8
- add frr/extras/el9

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/251
---
 hieradata/roles/infra/reposync/syncer.yaml | 28 ++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index 2ccd0ae..b672ee2 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -122,6 +122,34 @@ profiles::reposync::repos_list:
     release: 'el9'
     baseurl: 'https://download.docker.com/linux/centos/9/x86_64/stable/'
     gpgkey: 'https://download.docker.com/linux/centos/gpg'
+  frr_stable_el8:
+    repository: 'stable'
+    description: 'FRR Stable EL8'
+    osname: 'frr'
+    release: 'el8'
+    baseurl: 'https://rpm.frrouting.org/repo/el8/frr/'
+    gpgkey: 'https://packagerepo.service.consul/frr/gpg/RPM-GPG-KEY-FRR'
+  frr_extras_el8:
+    repository: 'extras'
+    description: 'FRR Extras EL8'
+    osname: 'frr'
+    release: 'el8'
+    baseurl: 'https://rpm.frrouting.org/repo/el8/extras/'
+    gpgkey: 'https://packagerepo.service.consul/frr/gpg/RPM-GPG-KEY-FRR'
+  frr_stable_el9:
+    repository: 'stable'
+    description: 'FRR Stable EL9'
+    osname: 'frr'
+    release: 'el9'
+    baseurl: 'https://rpm.frrouting.org/repo/el9/frr/'
+    gpgkey: 'https://packagerepo.service.consul/frr/gpg/RPM-GPG-KEY-FRR'
+  frr_extras_el9:
+    repository: 'extras'
+    description: 'FRR Extras el9'
+    osname: 'frr'
+    release: 'el9'
+    baseurl: 'https://rpm.frrouting.org/repo/el9/extras/'
+    gpgkey: 'https://packagerepo.service.consul/frr/gpg/RPM-GPG-KEY-FRR'
   k8s_1.32:
     repository: '1.32'
     description: 'Kubernetes 1.32'

From 278f8001b0f33e2719b034ea5fa69269479555f2 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 18 Apr 2025 21:21:23 +1000
Subject: [PATCH 40/89] feat: add frr synced repo (#252)

- add frr repo to incus hosts

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/252
---
 hieradata/roles/infra/incus/node.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index cb70475..f624b35 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -42,6 +42,20 @@ profiles::consul::client::node_rules:
 
 # additional repos
 profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
   zfs-kmod:
     name: zfs-kmod
     descr: zfs-kmod repository

From bfda2b628bf813f9770a01a6ecd08ae066e64cda Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 21 Apr 2025 18:40:17 +1000
Subject: [PATCH 41/89] feat: enable ip forwarding for gitea runners (#253)

- required to enable docker containers reach git service

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/253
---
 hieradata/roles/infra/git/runner.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/hieradata/roles/infra/git/runner.yaml b/hieradata/roles/infra/git/runner.yaml
index f94954f..851c6df 100644
--- a/hieradata/roles/infra/git/runner.yaml
+++ b/hieradata/roles/infra/git/runner.yaml
@@ -45,3 +45,10 @@ profiles::gitea::runner::config:
     force_rebuild: false
   host:
     workdir_parent: "%{hiera('profiles::gitea::runner::home')}/.cache/act"
+
+# enable ip forwarding for docker containers
+sysctl::base::values:
+  net.ipv4.conf.all.forwarding:
+    value: '1'
+  net.ipv6.conf.all.forwarding:
+    value: '1'

From c24babe3099b6880f315bf4881451a860a6626eb Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 01:00:39 +1000
Subject: [PATCH 42/89] feat: add incus image host (#254)

- add role
- add consul service + checks
- manage the datavol as zfs
- insure the incus fact exists before attempting to read it

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/254
---
 hieradata/roles/infra/incus/imagehost.yaml    | 125 ++++++++++++++++++
 modules/incus/manifests/init.pp               |  22 ++-
 site/roles/manifests/infra/incus/imagehost.pp |  10 ++
 3 files changed, 152 insertions(+), 5 deletions(-)
 create mode 100644 hieradata/roles/infra/incus/imagehost.yaml
 create mode 100644 site/roles/manifests/infra/incus/imagehost.pp

diff --git a/hieradata/roles/infra/incus/imagehost.yaml b/hieradata/roles/infra/incus/imagehost.yaml
new file mode 100644
index 0000000..ef0ca71
--- /dev/null
+++ b/hieradata/roles/infra/incus/imagehost.yaml
@@ -0,0 +1,125 @@
+---
+hiera_include:
+  - incus
+  - zfs
+
+profiles::packages::include:
+  bridge-utils: {}
+  dnsmasq: {}
+
+profiles::pki::vault::alt_names:
+  - incus-images.service.consul
+  - incus-images.query.consul
+  - "incus-images.service.%{facts.country}-%{facts.region}.consul"
+
+profiles::ssh::sign::principals:
+  - incus-images.service.consul
+  - incus-images.query.consul
+  - "incus-images.service.%{facts.country}-%{facts.region}.consul"
+
+# configure consul service
+consul::services:
+  incus-images:
+    service_name: 'incus-images'
+    tags:
+      - 'incus'
+      - 'images'
+      - 'container'
+      - 'lxd'
+    address: "%{facts.networking.ip}"
+    port: 8443
+    checks:
+      - id: 'incus_https_check'
+        name: 'incus HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:8443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: incus-images
+    disposition: write
+
+# additional repos
+profiles::yum::global::repos:
+  zfs-kmod:
+    name: zfs-kmod
+    descr: zfs-kmod repository
+    target: /etc/yum.repos.d/zfs-kmod.repo
+    baseurl: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-openzfs-2022
+    mirrorlist: absent
+
+# zfs settings
+zfs::manage_repo: false
+zfs::zfs_arc_min: ~
+zfs::zfs_arc_max: 429496729 # 400MB
+zfs::zpools:
+  fastpool:
+    ensure: present
+    disk: /dev/vdb
+    ashift: 12
+zfs::datasets:
+  fastpool:
+    canmount: 'off'
+    acltype: posix
+    atime: 'off'
+    relatime: 'off'
+    compression: 'zstd'
+    xattr: 'sa'
+  fastpool/data:
+    canmount: 'on'
+    mountpoint: '/data'
+  fastpool/data/incus:
+    canmount: 'on'
+    mountpoint: '/data/incus'
+
+# manage incus
+incus::init: true
+incus::server_port: 8443
+incus::storage_images_volume: fastpool/imagestore
+
+# add sysadmin to incus-admin group
+profiles::accounts::sysadmin::extra_groups:
+  - incus-admin
+
+# sysctl recommendations
+sysctl::base::values:
+  fs.aio-max-nr:
+    value: '524288'
+  fs.inotify.max_queued_events:
+    value: '1048576'
+  fs.inotify.max_user_instances:
+    value: '1048576'
+  fs.inotify.max_user_watches:
+    value: '1048576'
+  kernel.dmesg_restrict:
+    value: '1'
+  kernel.keys.maxbytes:
+    value: '2000000'
+  kernel.keys.maxkeys:
+    value: '2000'
+  net.core.bpf_jit_limit:
+    value: '1000000000'
+  net.ipv4.neigh.default.gc_thresh3:
+    value: '8192'
+  net.ipv6.neigh.default.gc_thresh3:
+    value: '8192'
+  vm.max_map_count:
+    value: '262144'
+  net.ipv4.conf.all.forwarding:
+    value: '1'
+  net.ipv6.conf.all.forwarding:
+    value: '1'
+
+# limits.d recommendations
+limits::entries:
+  '*/nofile':
+    both: 1048576
+  'root/nofile':
+    both: 1048576
+  '*/memlock':
+    both: unlimited
+  'root/memlock':
+    both: unlimited
diff --git a/modules/incus/manifests/init.pp b/modules/incus/manifests/init.pp
index 6933997..077de8f 100644
--- a/modules/incus/manifests/init.pp
+++ b/modules/incus/manifests/init.pp
@@ -9,6 +9,7 @@ class incus (
   String        $bridge       = 'incusbr0',
   Stdlib::Port        $server_port = 8443,
   Stdlib::IP::Address $server_addr = $facts['networking']['ip'],
+  Optional[String] $storage_images_volume = undef,
 ) {
 
   package { $packages:
@@ -55,11 +56,22 @@ class incus (
     }
   }
 
-  # set core.https_address
-  if $facts['incus']['config']['core.https_address'] != "${server_addr}:${server_port}" {
-    exec { 'incus_config_set_core_https_address':
-      path    => ['/bin', '/usr/bin'],
-      command => "incus config set core.https_address ${server_addr}:${server_port}",
+  if $facts['incus'] and $facts['incus']['config'] {
+    # set core.https_address
+    if $facts['incus']['config']['core.https_address'] != "${server_addr}:${server_port}" {
+      exec { 'incus_config_set_core_https_address':
+        path    => ['/bin', '/usr/bin'],
+        command => "incus config set core.https_address ${server_addr}:${server_port}",
+      }
+    }
+    # set storage.images_volume # path to store images
+    if $storage_images_volume {
+      if $facts['incus']['config']['storage.images_volume'] != $storage_images_volume {
+        exec { 'incus_config_set_storage_images_volume':
+          path    => ['/bin', '/usr/bin'],
+          command => "incus config set storage.images_volume ${storage_images_volume}",
+        }
+      }
     }
   }
 }
diff --git a/site/roles/manifests/infra/incus/imagehost.pp b/site/roles/manifests/infra/incus/imagehost.pp
new file mode 100644
index 0000000..26a4716
--- /dev/null
+++ b/site/roles/manifests/infra/incus/imagehost.pp
@@ -0,0 +1,10 @@
+# a role to deploy a incus image server
+class roles::infra::incus::imagehost {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}

From 2321186ad58003094d77066b460e5a9bd90b2cba Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 16:51:31 +1000
Subject: [PATCH 43/89] neoloc/mpls_ldp_frr (#255)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/255
---
 hieradata/roles/infra/incus/node.yaml    | 24 +++++++++++-
 hieradata/roles/infra/puppet/master.yaml |  7 ++++
 modules/frrouting/manifests/init.pp      | 26 ++++++++++++-
 modules/frrouting/templates/daemons.erb  |  1 +
 modules/frrouting/templates/frr.conf.erb | 18 +++++++++
 modules/libs/lib/facter/subnet_facts.rb  |  2 +-
 site/profiles/manifests/selinux/frr.pp   | 47 ++++++++++++++++++++++++
 7 files changed, 121 insertions(+), 4 deletions(-)
 create mode 100644 site/profiles/manifests/selinux/frr.pp

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index f624b35..2fa57f9 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -1,5 +1,6 @@
 ---
 hiera_include:
+  - profiles::selinux::frr
   - frrouting
   - incus
   - zfs
@@ -109,8 +110,15 @@ frrouting::ospfd_interfaces:
     area: 0.0.0.0
   loopback2:
     area: 0.0.0.0
-  br10:
-    area: 0.0.0.0
+frrouting::mpls_te_enabled: true
+frrouting::mpls_ldp_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::mpls_ldp_transport_addr: "%{hiera('networking_loopback0_ip')}"
+frrouting::mpls_ldp_interfaces:
+  - enp2s0
+  - enp3s0
+frrouting::daemons:
+  ldpd: true
+  ospfd: true
 
 # add loopback interfaces to ssh list
 ssh::server::options:
@@ -179,6 +187,18 @@ sysctl::base::values:
     value: '1'
   net.ipv6.conf.all.forwarding:
     value: '1'
+  net.ipv4.tcp_l3mdev_accept:
+    value: '0'
+  net.ipv4.conf.default.rp_filter:
+    value: '0'
+  net.ipv4.conf.all.rp_filter:
+    value: '0'
+  net.mpls.platform_labels:
+    value: '1048575'
+  net.mpls.conf.enp2s0.input:
+    value: '1'
+  net.mpls.conf.enp3s0.input:
+    value: '1'
 
 # limits.d recommendations
 limits::entries:
diff --git a/hieradata/roles/infra/puppet/master.yaml b/hieradata/roles/infra/puppet/master.yaml
index c9a7376..37ebf4f 100644
--- a/hieradata/roles/infra/puppet/master.yaml
+++ b/hieradata/roles/infra/puppet/master.yaml
@@ -5,6 +5,13 @@ profiles::puppet::autosign::subnet_ranges:
   - '198.18.15.0/24'
   - '198.18.16.0/24'
   - '198.18.17.0/24'
+  - '198.18.20.0/24'
+  - '198.18.24.0/24'
+  - '198.18.25.0/24'
+  - '198.18.26.0/24'
+  - '198.18.27.0/24'
+  - '198.18.28.0/24'
+  - '198.18.29.0/24'
 
 profiles::puppet::autosign::domains:
   - '*.main.unkin.net'
diff --git a/modules/frrouting/manifests/init.pp b/modules/frrouting/manifests/init.pp
index b5acb3d..007aa37 100644
--- a/modules/frrouting/manifests/init.pp
+++ b/modules/frrouting/manifests/init.pp
@@ -10,12 +10,17 @@ class frrouting (
   Array[String] $ospfd_redistribute = [],
   Array[String] $ospfd_networks     = [],
   Boolean $ospfd_default_originate_always = false,
+  Boolean $mpls_te_enabled                   = false,
+  Optional[String] $mpls_ldp_router_id       = undef,
+  Optional[String] $mpls_ldp_transport_addr  = undef,
+  Array[String] $mpls_ldp_interfaces         = [],
 ) {
 
   $daemons_defaults = {
     'bgpd'    => false,
     'ospfd'   => true,
     'ospf6d'  => false,
+    'ldpd'    => false,
     'ripd'    => false,
     'ripngd'  => false,
     'isisd'   => false,
@@ -32,7 +37,7 @@ class frrouting (
     'staticd' => false,
   }
 
-  $daemons_merged = merge($daemons, $daemons_defaults)
+  $daemons_merged = merge($daemons_defaults, $daemons)
 
   if $manage_package {
     package { $package_name:
@@ -62,4 +67,23 @@ class frrouting (
       hasrestart => true,
     }
   }
+
+  if $mpls_ldp_router_id and $mpls_ldp_transport_addr and !empty($mpls_ldp_interfaces) {
+    file { '/etc/modules-load.d/mpls_ldp_modules.conf':
+      ensure  => file,
+      content => @(EOT/L),
+        # Load MPLS Kernel Modules
+        mpls_router
+        mpls_iptunnel
+        | EOT
+    }
+
+    ['mpls_router', 'mpls_iptunnel'].each |$mod| {
+      exec { "load_${mod}":
+        command => "/sbin/modprobe ${mod}",
+        unless  => "/sbin/lsmod | /bin/grep -q ^${mod}",
+        path    => ['/sbin', '/bin', '/usr/sbin', '/usr/bin'],
+      }
+    }
+  }
 }
diff --git a/modules/frrouting/templates/daemons.erb b/modules/frrouting/templates/daemons.erb
index 846b339..09baa52 100644
--- a/modules/frrouting/templates/daemons.erb
+++ b/modules/frrouting/templates/daemons.erb
@@ -12,6 +12,7 @@ zebra_options="  -A 127.0.0.1 -s 90000000"
 bgpd_options="   -A 127.0.0.1"
 ospfd_options="  -A 127.0.0.1"
 ospf6d_options=" -A ::1"
+ldpd_options="   -A 127.0.0.1"
 ripd_options="   -A 127.0.0.1"
 ripngd_options=" -A ::1"
 isisd_options="  -A 127.0.0.1"
diff --git a/modules/frrouting/templates/frr.conf.erb b/modules/frrouting/templates/frr.conf.erb
index f1638fd..4e6668b 100644
--- a/modules/frrouting/templates/frr.conf.erb
+++ b/modules/frrouting/templates/frr.conf.erb
@@ -24,4 +24,22 @@ router ospf
 <% if @ospfd_default_originate_always -%>
  default-information originate always
 <% end -%>
+<% if @mpls_te_enabled -%>
+ capability opaque
+ mpls-te on
+ mpls-te router-address <%= @ospfd_router_id %>
+ mpls-te inter-as area 0.0.0.0
+<% end -%>
 exit
+<% if @mpls_ldp_router_id and @mpls_ldp_transport_addr and @mpls_ldp_interfaces.any? -%>
+mpls ldp
+ router-id <%= @mpls_ldp_router_id %>
+ address-family ipv4
+  discovery transport-address <%= @mpls_ldp_transport_addr %>
+<% @mpls_ldp_interfaces.each do |iface| -%>
+  interface <%= iface %>
+  exit
+<% end -%>
+ exit-address-family
+exit
+<% end -%>
diff --git a/modules/libs/lib/facter/subnet_facts.rb b/modules/libs/lib/facter/subnet_facts.rb
index 28325fd..6bc2886 100644
--- a/modules/libs/lib/facter/subnet_facts.rb
+++ b/modules/libs/lib/facter/subnet_facts.rb
@@ -12,7 +12,7 @@ class SubnetAttributes
     '198.18.17.0/24' => { environment: 'prod', region: 'drw1', country: 'au' },
     '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' },
     '198.18.19.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # loopbacks
-    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # RESERVED
+    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # MPLS CORE BLOCKS
     '198.18.21.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # physical network 2.5gbe
     '198.18.22.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph cluster
     '198.18.23.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph public
diff --git a/site/profiles/manifests/selinux/frr.pp b/site/profiles/manifests/selinux/frr.pp
new file mode 100644
index 0000000..65bcb46
--- /dev/null
+++ b/site/profiles/manifests/selinux/frr.pp
@@ -0,0 +1,47 @@
+# this is a modification to frr-selinux that ships with EL9, adding support for frr10
+class profiles::selinux::frr {
+
+  $frr_te_content = @("EOF")
+    module frr_local 1.0;
+
+    require {
+      type frr_t;
+      type initrc_t;
+      type kernel_t;
+      type var_run_t;
+      type frr_tmp_t;
+      type frr_var_run_t;
+      type init_t;
+      class unix_stream_socket connectto;
+      class system module_request;
+      class sock_file { getattr write };
+      class dir { add_name write };
+      class file { create write open };
+      class process setpgid;
+    }
+
+    #============= frr_t ==============
+    allow frr_t initrc_t:unix_stream_socket connectto;
+    allow frr_t kernel_t:system module_request;
+    allow frr_t var_run_t:sock_file { getattr write };
+
+    #============= init_t ==============
+    allow init_t frr_tmp_t:dir add_name;
+    allow init_t frr_var_run_t:dir { write add_name };
+    allow init_t frr_var_run_t:file { create open write };
+    allow init_t self:process setpgid;
+  | EOF
+
+  selinux::module { 'frr_local':
+    ensure     => 'present',
+    content_te => $frr_te_content,
+    builder    => 'simple',
+    before     => Service['frr'],
+  }
+
+  selboolean { 'domain_can_mmap_files':
+    value      => 'on',
+    persistent => true,
+    before     => Service['frr'],
+  }
+}

From bc5bd11f5e15b3ac0e7ff96f75c7f358d889511a Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 21:18:59 +1000
Subject: [PATCH 44/89] feat: disable cobbler cache (#256)

- this is required to resolve issues with terraform deploying cobbler
  settings

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/256
---
 site/profiles/manifests/cobbler/config.pp         | 1 +
 site/profiles/manifests/cobbler/params.pp         | 1 +
 site/profiles/templates/cobbler/settings.yaml.erb | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/site/profiles/manifests/cobbler/config.pp b/site/profiles/manifests/cobbler/config.pp
index c736be8..3042d98 100644
--- a/site/profiles/manifests/cobbler/config.pp
+++ b/site/profiles/manifests/cobbler/config.pp
@@ -11,6 +11,7 @@ class profiles::cobbler::config {
   $service_cname = $profiles::cobbler::params::service_cname
   $next_server = $profiles::cobbler::params::next_server
   $server = $profiles::cobbler::params::server
+  $cache_enabled = $profiles::cobbler::params::cache_enabled
 
   # manage the cobbler settings file
   file { '/etc/cobbler/settings.yaml':
diff --git a/site/profiles/manifests/cobbler/params.pp b/site/profiles/manifests/cobbler/params.pp
index 877f986..2b51acf 100644
--- a/site/profiles/manifests/cobbler/params.pp
+++ b/site/profiles/manifests/cobbler/params.pp
@@ -9,6 +9,7 @@ class profiles::cobbler::params (
   String  $next_server   = $::facts['networking']['ip'],
   Boolean $pxe_just_once = true,
   Boolean $is_cobbler_master = false,
+  Boolean $cache_enabled = false,
   Array   $packages = [
       'cobbler',
       'cobbler3.2-web',
diff --git a/site/profiles/templates/cobbler/settings.yaml.erb b/site/profiles/templates/cobbler/settings.yaml.erb
index 1869444..135d431 100644
--- a/site/profiles/templates/cobbler/settings.yaml.erb
+++ b/site/profiles/templates/cobbler/settings.yaml.erb
@@ -59,7 +59,7 @@ build_reporting_ignorelist: []
 # use cases like writing out large numbers of records. There is a known issue with cache and remote XMLRPC API calls.
 # If you will use Cobbler with config management or infrastructure-as-code tools such as Terraform, it is recommended
 # to disable by setting to false.
-cache_enabled: true
+cache_enabled: <%= @cache_enabled %>
 
 # Cheetah-language autoinstall templates can import Python modules. While this is a useful feature, it is not safe to
 # allow them to import anything they want. This whitelists which modules can be imported through Cheetah. Users can

From a21c1b36977f5cf6071eb4cf99aa6fa89da2b1b9 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 21:25:00 +1000
Subject: [PATCH 45/89] Adding
 hieradata/node/ausyd1nxvm1072.main.unkin.net.yaml (#257)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/257
---
 hieradata/nodes/ausyd1nxvm1072.main.unkin.net.yaml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 hieradata/nodes/ausyd1nxvm1072.main.unkin.net.yaml

diff --git a/hieradata/nodes/ausyd1nxvm1072.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm1072.main.unkin.net.yaml
new file mode 100644
index 0000000..97bf7f6
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm1072.main.unkin.net.yaml
@@ -0,0 +1,7 @@
+---
+networking::interfaces:
+  eth0:
+    ipaddress: 198.18.13.82
+networking::routes:
+  default:
+    gateway: 198.18.13.254
\ No newline at end of file

From 9dcaafb8babefa8b9ecf5a2e9f8d896e914fce89 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 23:03:01 +1000
Subject: [PATCH 46/89] feat: lxc updates (#258)

- add virtual/lxc.yaml
- add crypto crypto-policies-scripts
- ensure ssh::server is managed

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/258
---
 hieradata/os/AlmaLinux/all_releases.yaml | 1 +
 hieradata/virtual/lxc.yaml               | 1 +
 site/profiles/manifests/ssh/service.pp   | 2 ++
 3 files changed, 4 insertions(+)
 create mode 100644 hieradata/virtual/lxc.yaml

diff --git a/hieradata/os/AlmaLinux/all_releases.yaml b/hieradata/os/AlmaLinux/all_releases.yaml
index 5a09a26..251649b 100644
--- a/hieradata/os/AlmaLinux/all_releases.yaml
+++ b/hieradata/os/AlmaLinux/all_releases.yaml
@@ -9,6 +9,7 @@ hiera_include:
   - profiles::almalinux::base
 
 profiles::packages::include:
+  crypto-policies-scripts: {}
   lzo: {}
   policycoreutils: {}
   unar: {}
diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
new file mode 100644
index 0000000..ed97d53
--- /dev/null
+++ b/hieradata/virtual/lxc.yaml
@@ -0,0 +1 @@
+---
diff --git a/site/profiles/manifests/ssh/service.pp b/site/profiles/manifests/ssh/service.pp
index e334016..c75a625 100644
--- a/site/profiles/manifests/ssh/service.pp
+++ b/site/profiles/manifests/ssh/service.pp
@@ -2,6 +2,8 @@
 # saz-ssh manages the service, this is just some additional stuff
 class profiles::ssh::service {
 
+  include ssh::server
+
   # set sshd to start
   systemd::manage_dropin { 'after-network-online.conf':
     ensure     => present,

From ecce93bedbceb6afd1c7e9ca5cbdecb2a23a96e7 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 23:18:45 +1000
Subject: [PATCH 47/89] feat: lxc cannot use chronyd (#259)

- ensure lxc nodes do not attempt to install chronyd
- ensure chrony is removed

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/259
---
 hieradata/virtual/lxc.yaml      | 3 +++
 site/profiles/manifests/base.pp | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
index ed97d53..8309995 100644
--- a/hieradata/virtual/lxc.yaml
+++ b/hieradata/virtual/lxc.yaml
@@ -1 +1,4 @@
 ---
+profiles::packages::include:
+  chrony:
+    ensure: absent
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index 4ec5150..46df942 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -28,7 +28,9 @@ class profiles::base (
     include profiles::base::groups
     include profiles::base::root
     include profiles::accounts::sysadmin
-    include profiles::ntp::client
+    if $facts['virtual'] != 'lxc' {
+      include profiles::ntp::client
+    }
     include profiles::dns::base
     include profiles::pki::vault
     include profiles::ssh::sign

From 463abe4b9d5efe67be58479b0f54ef8c10e35b4d Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 24 Apr 2025 23:48:34 +1000
Subject: [PATCH 48/89] feat: add reverse dns zones for incus (#260)

- add reverse dns zones for incus hosts
- update acls for openresolver

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/260
---
 hieradata/roles/infra/dns/master.yaml   | 85 +++++++++++++++++++++++++
 hieradata/roles/infra/dns/resolver.yaml | 11 ++++
 2 files changed, 96 insertions(+)

diff --git a/hieradata/roles/infra/dns/master.yaml b/hieradata/roles/infra/dns/master.yaml
index e9b81b7..c83c101 100644
--- a/hieradata/roles/infra/dns/master.yaml
+++ b/hieradata/roles/infra/dns/master.yaml
@@ -9,6 +9,14 @@ profiles::dns::master::acls:
       - 198.18.15.0/24
       - 198.18.16.0/24
       - 198.18.17.0/24
+      - 198.18.19.0/24
+      - 198.18.20.0/24
+      - 198.18.24.0/24
+      - 198.18.25.0/24
+      - 198.18.26.0/24
+      - 198.18.27.0/24
+      - 198.18.28.0/24
+      - 198.18.29.0/24
 
 profiles::dns::master::zones:
   main.unkin.net:
@@ -47,6 +55,72 @@ profiles::dns::master::zones:
     dynamic: false
     ns_notify: true
     source: '/var/named/sources/17.18.198.in-addr.arpa.conf'
+  19.18.198.in-addr.arpa:
+    domain: '19.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/19.18.198.in-addr.arpa.conf'
+  20.18.198.in-addr.arpa:
+    domain: '20.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/20.18.198.in-addr.arpa.conf'
+  21.18.198.in-addr.arpa:
+    domain: '21.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/21.18.198.in-addr.arpa.conf'
+  22.18.198.in-addr.arpa:
+    domain: '22.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/22.18.198.in-addr.arpa.conf'
+  23.18.198.in-addr.arpa:
+    domain: '23.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/23.18.198.in-addr.arpa.conf'
+  24.18.198.in-addr.arpa:
+    domain: '24.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/24.18.198.in-addr.arpa.conf'
+  25.18.198.in-addr.arpa:
+    domain: '25.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/25.18.198.in-addr.arpa.conf'
+  26.18.198.in-addr.arpa:
+    domain: '26.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/26.18.198.in-addr.arpa.conf'
+  27.18.198.in-addr.arpa:
+    domain: '27.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/27.18.198.in-addr.arpa.conf'
+  28.18.198.in-addr.arpa:
+    domain: '28.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/28.18.198.in-addr.arpa.conf'
+  29.18.198.in-addr.arpa:
+    domain: '29.18.198.in-addr.arpa'
+    zone_type: 'master'
+    dynamic: false
+    ns_notify: true
+    source: '/var/named/sources/29.18.198.in-addr.arpa.conf'
 
 profiles::dns::master::views:
   master-zones:
@@ -58,6 +132,17 @@ profiles::dns::master::views:
       - 15.18.198.in-addr.arpa
       - 16.18.198.in-addr.arpa
       - 17.18.198.in-addr.arpa
+      - 19.18.198.in-addr.arpa
+      - 20.18.198.in-addr.arpa
+      - 21.18.198.in-addr.arpa
+      - 22.18.198.in-addr.arpa
+      - 23.18.198.in-addr.arpa
+      - 24.18.198.in-addr.arpa
+      - 25.18.198.in-addr.arpa
+      - 26.18.198.in-addr.arpa
+      - 27.18.198.in-addr.arpa
+      - 28.18.198.in-addr.arpa
+      - 29.18.198.in-addr.arpa
     match_clients:
       - acl-main.unkin.net
 
diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index f978202..9ec4add 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -93,6 +93,17 @@ profiles::dns::resolver::views:
       - 15.18.198.in-addr.arpa-forward
       - 16.18.198.in-addr.arpa-forward
       - 17.18.198.in-addr.arpa-forward
+      - 19.18.198.in-addr.arpa-forward
+      - 20.18.198.in-addr.arpa-forward
+      - 21.18.198.in-addr.arpa-forward
+      - 22.18.198.in-addr.arpa-forward
+      - 23.18.198.in-addr.arpa-forward
+      - 24.18.198.in-addr.arpa-forward
+      - 25.18.198.in-addr.arpa-forward
+      - 26.18.198.in-addr.arpa-forward
+      - 27.18.198.in-addr.arpa-forward
+      - 28.18.198.in-addr.arpa-forward
+      - 29.18.198.in-addr.arpa-forward
       - 8.10.10.in-addr.arpa-forward
       - 16.10.10.in-addr.arpa-forward
       - 20.10.10.in-addr.arpa-forward

From 762d980ea8ef78e02efd5efb87e71c2cc37cbd8f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 25 Apr 2025 01:01:47 +1000
Subject: [PATCH 49/89] feat: update dns resolver zone management (#261)

- move zones to common role path
- specify forwarders for each zone in region based hiera

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/261
---
 .../au/region/drw1/infra/dns/resolver.yaml    | 58 ++----------
 .../au/region/syd1/infra/dns/resolver.yaml    | 58 ++----------
 hieradata/roles/infra/dns/resolver.yaml       | 90 +++++++++++++++++++
 3 files changed, 104 insertions(+), 102 deletions(-)

diff --git a/hieradata/country/au/region/drw1/infra/dns/resolver.yaml b/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
index 157667c..ae1582f 100644
--- a/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
@@ -1,52 +1,8 @@
 ---
-profiles::dns::resolver::zones:
-  main.unkin.net-forward:
-    domain: 'main.unkin.net'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  13.18.198.in-addr.arpa-forward:
-    domain: '13.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  14.18.198.in-addr.arpa-forward:
-    domain: '14.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  15.18.198.in-addr.arpa-forward:
-    domain: '15.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  16.18.198.in-addr.arpa-forward:
-    domain: '16.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  17.18.198.in-addr.arpa-forward:
-    domain: '17.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.23
-      - 198.18.17.24
-    forward: 'only'
-  consul-forward:
-    domain: 'consul'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.17.34
-      - 198.18.17.35
-      - 198.18.17.36
-    forward: 'only'
+profiles_dns_upstream_forwarder_unkin:
+  - 198.18.17.23
+  - 198.18.17.24
+profiles_dns_upstream_forwarder_consul:
+  - 198.18.17.34
+  - 198.18.17.35
+  - 198.18.17.36
diff --git a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
index 088f065..b26491e 100644
--- a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
@@ -1,52 +1,8 @@
 ---
-profiles::dns::resolver::zones:
-  main.unkin.net-forward:
-    domain: 'main.unkin.net'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  13.18.198.in-addr.arpa-forward:
-    domain: '13.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  14.18.198.in-addr.arpa-forward:
-    domain: '14.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  15.18.198.in-addr.arpa-forward:
-    domain: '15.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  16.18.198.in-addr.arpa-forward:
-    domain: '16.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  17.18.198.in-addr.arpa-forward:
-    domain: '17.18.198.in-addr.arpa'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.14
-      - 198.18.13.15
-    forward: 'only'
-  consul-forward:
-    domain: 'consul'
-    zone_type: 'forward'
-    forwarders:
-      - 198.18.13.19
-      - 198.18.13.20
-      - 198.18.13.21
-    forward: 'only'
+profiles_dns_upstream_forwarder_unkin:
+  - 198.18.13.14
+  - 198.18.13.15
+profiles_dns_upstream_forwarder_consul:
+  - 198.18.13.19
+  - 198.18.13.20
+  - 198.18.13.21
diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index 9ec4add..f94eb93 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -78,6 +78,96 @@ profiles::dns::resolver::zones:
       - 10.10.16.32
       - 10.10.16.33
     forward: 'only'
+  main.unkin.net-forward:
+    domain: 'main.unkin.net'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  13.18.198.in-addr.arpa-forward:
+    domain: '13.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  14.18.198.in-addr.arpa-forward:
+    domain: '14.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  15.18.198.in-addr.arpa-forward:
+    domain: '15.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  16.18.198.in-addr.arpa-forward:
+    domain: '16.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  17.18.198.in-addr.arpa-forward:
+    domain: '17.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  19.18.198.in-addr.arpa-forward:
+    domain: '19.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  20.18.198.in-addr.arpa-forward:
+    domain: '20.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  21.18.198.in-addr.arpa-forward:
+    domain: '21.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  22.18.198.in-addr.arpa-forward:
+    domain: '22.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  23.18.198.in-addr.arpa-forward:
+    domain: '23.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  24.18.198.in-addr.arpa-forward:
+    domain: '24.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  25.18.198.in-addr.arpa-forward:
+    domain: '25.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  26.18.198.in-addr.arpa-forward:
+    domain: '26.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  27.18.198.in-addr.arpa-forward:
+    domain: '27.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  28.18.198.in-addr.arpa-forward:
+    domain: '28.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  29.18.198.in-addr.arpa-forward:
+    domain: '29.18.198.in-addr.arpa'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
+    forward: 'only'
+  consul-forward:
+    domain: 'consul'
+    zone_type: 'forward'
+    forwarders: "%{alias('profiles_dns_upstream_forwarder_consul')}"
+    forward: 'only'
 
 profiles::dns::resolver::views:
   openforwarder:

From 78f4d2a88f9b56fdb9e61ebf49012fd6c98fdb80 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 26 Apr 2025 00:39:23 +1000
Subject: [PATCH 50/89] feat: cleanup mpls configuration (#262)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/262
---
 hieradata/roles/infra/incus/node.yaml | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 2fa57f9..a5cadbc 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -110,14 +110,7 @@ frrouting::ospfd_interfaces:
     area: 0.0.0.0
   loopback2:
     area: 0.0.0.0
-frrouting::mpls_te_enabled: true
-frrouting::mpls_ldp_router_id: "%{hiera('networking_loopback0_ip')}"
-frrouting::mpls_ldp_transport_addr: "%{hiera('networking_loopback0_ip')}"
-frrouting::mpls_ldp_interfaces:
-  - enp2s0
-  - enp3s0
 frrouting::daemons:
-  ldpd: true
   ospfd: true
 
 # add loopback interfaces to ssh list
@@ -193,12 +186,6 @@ sysctl::base::values:
     value: '0'
   net.ipv4.conf.all.rp_filter:
     value: '0'
-  net.mpls.platform_labels:
-    value: '1048575'
-  net.mpls.conf.enp2s0.input:
-    value: '1'
-  net.mpls.conf.enp3s0.input:
-    value: '1'
 
 # limits.d recommendations
 limits::entries:

From e4166c6b140ddf624414c4a7e0ee6dd8c0ebd83b Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 26 Apr 2025 17:28:57 +1000
Subject: [PATCH 51/89] feat: lxc compatability with datavol (#263)

- lxc doesnt mount block devices, just check for mountpoint

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/263
---
 site/profiles/manifests/base/datavol.pp | 74 ++++++++++++++-----------
 1 file changed, 42 insertions(+), 32 deletions(-)

diff --git a/site/profiles/manifests/base/datavol.pp b/site/profiles/manifests/base/datavol.pp
index 5cb2a12..cf37f8e 100644
--- a/site/profiles/manifests/base/datavol.pp
+++ b/site/profiles/manifests/base/datavol.pp
@@ -2,6 +2,9 @@
 #
 # This class manages the creation of a logical volume using the `lvm::volume` definition.
 #
+# For LXC hosts, this is replaced with a mount added from the host os. This class will simply check the
+# mountpoint exists.
+#
 # Parameters:
 #   $ensure   - Ensure whether the logical volume is present or not. Defaults to 'present'.
 #   $vg       - Volume group name. No default.
@@ -25,33 +28,48 @@ class profiles::base::datavol (
   ]] $mount_options = ['noatime', 'nodiratime'],
 ) {
 
-  # Ensure the physical volume exists
-  physical_volume { $pv:
-    ensure => $ensure,
-    before => Volume_group[$vg],
-  }
+  if $facts['virtual'] != 'lxc' {
 
-  # Ensure the volume group exists
-  volume_group { $vg:
-    ensure           => $ensure,
-    physical_volumes => [$pv],
-    before           => Logical_volume[$lv],
-  }
+    # Ensure the physical volume exists
+    physical_volume { $pv:
+      ensure => $ensure,
+      before => Volume_group[$vg],
+    }
 
-  # Ensure the logical volume exists
-  logical_volume { $lv:
-    ensure       => $ensure,
-    volume_group => $vg,
-    size         => $size,
-    before       => Filesystem["/dev/${vg}/${lv}"],
-  }
+    # Ensure the volume group exists
+    volume_group { $vg:
+      ensure           => $ensure,
+      physical_volumes => [$pv],
+      before           => Logical_volume[$lv],
+    }
 
-  # Ensure the filesystem is created on the logical volume
-  filesystem { "/dev/${vg}/${lv}":
-    ensure  => $ensure,
-    fs_type => $fstype,
-    require => Logical_volume[$lv],
-    before  => Mount[$mount],
+    # Ensure the logical volume exists
+    logical_volume { $lv:
+      ensure       => $ensure,
+      volume_group => $vg,
+      size         => $size,
+      before       => Filesystem["/dev/${vg}/${lv}"],
+    }
+
+    # Ensure the filesystem is created on the logical volume
+    filesystem { "/dev/${vg}/${lv}":
+      ensure  => $ensure,
+      fs_type => $fstype,
+      require => Logical_volume[$lv],
+      before  => Mount[$mount],
+    }
+
+    # Ensure the logical volume is mounted at the desired location
+    mount { $mount:
+      ensure  => $mountstate,
+      device  => "/dev/${vg}/${lv}",
+      fstype  => $fstype,
+      options => $mount_options.join(','),
+      require => [
+        Filesystem["/dev/${vg}/${lv}"],
+        File[$mount]
+      ],
+    }
   }
 
   # Ensure the mountpath exists
@@ -62,12 +80,4 @@ class profiles::base::datavol (
     mode   => '0755',
   }
 
-  # Ensure the logical volume is mounted at the desired location
-  mount { $mount:
-    ensure  => $mountstate,
-    device  => "/dev/${vg}/${lv}",
-    fstype  => $fstype,
-    options => $mount_options.join(','),
-    require => Filesystem["/dev/${vg}/${lv}"],
-  }
 }

From 496ed12a58f1e629102a3a9ceb39306ef32d1c1f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 26 Apr 2025 18:40:31 +1000
Subject: [PATCH 52/89] feat: change vault to use package install (#264)

- vault 18.2 rpm produced by rpmbuilder repo
- ensure the /etc/vault directory is managed
- ensure service file is managed by puppet
- ensure package comes from unkin repo (not hashicorp)
- disable_mlock as unprivileged containers cannot use mlock

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/264
---
 hieradata/common.yaml                   | 7 +++++++
 site/profiles/manifests/vault/server.pp | 5 -----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 89cec33..1df21ad 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -175,6 +175,13 @@ consul::install_method: 'package'
 consul::manage_repo: false
 consul::bin_dir: /usr/bin
 
+vault::install_method: 'repo'
+vault::manage_repo: false
+vault::bin_dir: /usr/bin
+vault::manage_service_file: true
+vault::manage_config_dir: true
+vault::disable_mlock: true
+
 profiles::dns::master::basedir: '/var/named/sources'
 profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
 profiles::dns::base::use_ns: 'region'
diff --git a/site/profiles/manifests/vault/server.pp b/site/profiles/manifests/vault/server.pp
index a27ef46..84398f4 100644
--- a/site/profiles/manifests/vault/server.pp
+++ b/site/profiles/manifests/vault/server.pp
@@ -6,10 +6,6 @@ class profiles::vault::server (
     Undef
   ]         $members_role   = undef,
   Array     $vault_servers  = [],
-  Enum[
-    'archive',
-    'repo'
-  ]         $install_method = 'archive',
   Boolean       $tls_disable        = false,
   Stdlib::Port  $client_port        = 8200,
   Stdlib::Port  $cluster_port       = 8201,
@@ -56,7 +52,6 @@ class profiles::vault::server (
 
     class { 'vault':
       manage_service     => false,
-      install_method     => $install_method,
       manage_storage_dir => $manage_storage_dir,
       enable_ui          => true,
       storage            => {

From 1e3ce0ec1c1c88d00c3c27962bc71efbfcd52ae1 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 26 Apr 2025 20:02:57 +1000
Subject: [PATCH 53/89] feat: dont set gid/uid for sysadmin (#265)

- sysadmin doesnt need to be a specific uid/gid, the next available
  uid/gid is fine

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/265
---
 site/profiles/manifests/accounts/sysadmin.pp | 2 --
 site/profiles/manifests/base/account.pp      | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/site/profiles/manifests/accounts/sysadmin.pp b/site/profiles/manifests/accounts/sysadmin.pp
index 0c9050d..8e5d6a6 100644
--- a/site/profiles/manifests/accounts/sysadmin.pp
+++ b/site/profiles/manifests/accounts/sysadmin.pp
@@ -15,8 +15,6 @@ class profiles::accounts::sysadmin(
 
   profiles::base::account {'sysadmin':
     username   => 'sysadmin',
-    uid        => 1000,
-    gid        => 1000,
     groups     => $groups,
     sshkeys    => $sshkeys,
     sudo_rules => ['sysadmin ALL=(ALL) NOPASSWD:ALL'],
diff --git a/site/profiles/manifests/base/account.pp b/site/profiles/manifests/base/account.pp
index 92011b4..e9dd48c 100644
--- a/site/profiles/manifests/base/account.pp
+++ b/site/profiles/manifests/base/account.pp
@@ -1,8 +1,8 @@
 # a wrapper for puppetlabs-account and saz-sudo
 define profiles::base::account (
   String $username,
-  Integer $uid,
-  Integer $gid = undef,
+  Optional[Integer] $uid = undef,
+  Optional[Integer] $gid = undef,
   Boolean $manage_home = true,
   Boolean $create_group = true,
   Boolean $purge_sshkeys = true,

From 9359b8902eed8296f47bd4035a74ba12f9de8969 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 26 Apr 2025 22:43:20 +1000
Subject: [PATCH 54/89] feat: vault mlock (#266)

- enable mlock by default
- disable mlock on lxd/incus nodes (lxc doesnt support it)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/266
---
 hieradata/common.yaml      | 2 +-
 hieradata/virtual/lxc.yaml | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 1df21ad..ae0a661 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -180,7 +180,7 @@ vault::manage_repo: false
 vault::bin_dir: /usr/bin
 vault::manage_service_file: true
 vault::manage_config_dir: true
-vault::disable_mlock: true
+vault::disable_mlock: false
 
 profiles::dns::master::basedir: '/var/named/sources'
 profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
index 8309995..f2d7929 100644
--- a/hieradata/virtual/lxc.yaml
+++ b/hieradata/virtual/lxc.yaml
@@ -2,3 +2,6 @@
 profiles::packages::include:
   chrony:
     ensure: absent
+
+# disable mlock for vault nodes on lxd/incus
+vault::disable_mlock: true

From 07b89ab737f87d2aa309adb04df94bd17be32964 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 28 Apr 2025 18:46:58 +1000
Subject: [PATCH 55/89] feat: enable terraform access to puppetca (#267)

- enable terraform to clean certificates

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/267
---
 site/profiles/manifests/puppet/server.pp      |   9 +
 .../templates/puppet/server/auth.conf.erb     | 266 ++++++++++++++++++
 2 files changed, 275 insertions(+)
 create mode 100644 site/profiles/templates/puppet/server/auth.conf.erb

diff --git a/site/profiles/manifests/puppet/server.pp b/site/profiles/manifests/puppet/server.pp
index 657bd41..94753ab 100644
--- a/site/profiles/manifests/puppet/server.pp
+++ b/site/profiles/manifests/puppet/server.pp
@@ -65,6 +65,15 @@ class profiles::puppet::server (
     notify  => Service['puppetserver'],
   }
 
+  file { '/etc/puppetlabs/puppetserver/conf.d/auth.conf':
+    ensure  => 'file',
+    content => template('profiles/puppet/server/auth.conf.erb'),
+    group   => 'root',
+    owner   => 'root',
+    mode    => '0644',
+    notify  => Service['puppetserver'],
+  }
+
   service { 'puppetserver':
     ensure     => running,
     enable     => true,
diff --git a/site/profiles/templates/puppet/server/auth.conf.erb b/site/profiles/templates/puppet/server/auth.conf.erb
new file mode 100644
index 0000000..9f36063
--- /dev/null
+++ b/site/profiles/templates/puppet/server/auth.conf.erb
@@ -0,0 +1,266 @@
+authorization: {
+    version: 1
+    rules: [
+        {
+            # Allow nodes to retrieve their own catalog
+            match-request: {
+                path: "^/puppet/v3/catalog/([^/]+)$"
+                type: regex
+                method: [get, post]
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs v3 catalog from agents"
+        },
+        {
+            # Allow services to retrieve catalogs on behalf of others
+            match-request: {
+                path: "^/puppet/v4/catalog/?$"
+                type: regex
+                method: post
+            }
+            deny: "*"
+            sort-order: 500
+            name: "puppetlabs v4 catalog for services"
+        },
+        {
+            # Allow nodes to retrieve the certificate they requested earlier
+            match-request: {
+                path: "/puppet-ca/v1/certificate/"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs certificate"
+        },
+        {
+            # Allow all nodes to access the certificate revocation list
+            match-request: {
+                path: "/puppet-ca/v1/certificate_revocation_list/ca"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs crl"
+        },
+        {
+            # Allow nodes to request a new certificate
+            match-request: {
+                path: "/puppet-ca/v1/certificate_request"
+                type: path
+                method: [get, put]
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs csr"
+        },
+        {
+            # Allow the CA CLI to access the certificate_status endpoint
+            match-request: {
+                path: "/puppet-ca/v1/certificate_status"
+                type: path
+                method: [get, put, delete]
+            }
+            allow: [
+              {
+                extensions: {
+                  pp_cli_auth: "true"
+                }
+              },
+              terraform
+            ]
+            sort-order: 500
+            name: "puppetlabs cert status"
+        },
+        {
+            match-request: {
+                path: "^/puppet-ca/v1/certificate_revocation_list$"
+                type: regex
+                method: put
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs CRL update"
+        },
+        {
+            # Allow the CA CLI to access the certificate_statuses endpoint
+            match-request: {
+                path: "/puppet-ca/v1/certificate_statuses"
+                type: path
+                method: get
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert statuses"
+        },
+        {
+            # Allow authenticated access to the CA expirations endpoint
+            match-request: {
+                path: "/puppet-ca/v1/expirations"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs CA cert and CRL expirations"
+        },
+        {
+            # Allow the CA CLI to access the certificate clean endpoint
+            match-request: {
+                path: "/puppet-ca/v1/clean"
+                type: path
+                method: put
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert clean"
+        },
+        {
+            # Allow unauthenticated access to the status service endpoint
+            match-request: {
+                path: "/status/v1/services"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs status service - full"
+        },
+        {
+            match-request: {
+                path: "/status/v1/simple"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs status service - simple"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/environments"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs environments"
+        },
+        {
+            # Allow nodes to access all file_bucket_files.  Note that access for
+            # the 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_bucket_file"
+                type: path
+                method: [get, head, post, put]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file bucket file"
+        },
+        {
+            # Allow nodes to access all file_content.  Note that access for the
+            # 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_content"
+                type: path
+                method: [get, post]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file content"
+        },
+        {
+            # Allow nodes to access all file_metadata.  Note that access for the
+            # 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_metadata"
+                type: path
+                method: [get, post]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file metadata"
+        },
+        {
+            # Allow nodes to retrieve only their own node definition
+            match-request: {
+                path: "^/puppet/v3/node/([^/]+)$"
+                type: regex
+                method: get
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs node"
+        },
+        {
+            # Allow nodes to store only their own reports
+            match-request: {
+                path: "^/puppet/v3/report/([^/]+)$"
+                type: regex
+                method: put
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs report"
+        },
+        {
+            # Allow nodes to update their own facts
+            match-request: {
+                path: "^/puppet/v3/facts/([^/]+)$"
+                type: regex
+                method: put
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs facts"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/static_file_content"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs static file content"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/tasks"
+                type: path
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppet tasks information"
+        },
+        {
+            # Deny everything else. This ACL is not strictly
+            # necessary, but illustrates the default policy
+            match-request: {
+                path: "/"
+                type: path
+            }
+            deny: "*"
+            sort-order: 999
+            name: "puppetlabs deny all"
+        }
+    ]
+}

From 2323ef77498af318cd013c1770cec0290001c8ec Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 28 Apr 2025 21:39:45 +1000
Subject: [PATCH 56/89] feat: postgresql15/postgresql17 (#268)

- add postgresql15 and 17 to reposync

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/268
---
 hieradata/roles/infra/reposync/syncer.yaml | 28 ++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/hieradata/roles/infra/reposync/syncer.yaml b/hieradata/roles/infra/reposync/syncer.yaml
index b672ee2..26d6d82 100644
--- a/hieradata/roles/infra/reposync/syncer.yaml
+++ b/hieradata/roles/infra/reposync/syncer.yaml
@@ -206,6 +206,20 @@ profiles::reposync::repos_list:
     release: 'rhel9'
     baseurl: 'https://download.postgresql.org/pub/repos/yum/common/redhat/rhel-9-x86_64/'
     gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  postgresql_rhel8_15:
+    repository: '15'
+    description: 'PostgreSQL 15 RHEL 8'
+    osname: 'postgresql'
+    release: 'rhel8'
+    baseurl: 'https://download.postgresql.org/pub/repos/yum/15/redhat/rhel-8-x86_64/'
+    gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  postgresql_rhel9_15:
+    repository: '15'
+    description: 'PostgreSQL 15 RHEL 9'
+    osname: 'postgresql'
+    release: 'rhel9'
+    baseurl: 'https://download.postgresql.org/pub/repos/yum/15/redhat/rhel-9-x86_64/'
+    gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
   postgresql_rhel8_16:
     repository: '16'
     description: 'PostgreSQL 16 RHEL 8'
@@ -220,6 +234,20 @@ profiles::reposync::repos_list:
     release: 'rhel9'
     baseurl: 'https://download.postgresql.org/pub/repos/yum/16/redhat/rhel-9-x86_64/'
     gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  postgresql_rhel8_17:
+    repository: '17'
+    description: 'PostgreSQL 17 RHEL 8'
+    osname: 'postgresql'
+    release: 'rhel8'
+    baseurl: 'https://download.postgresql.org/pub/repos/yum/17/redhat/rhel-8-x86_64/'
+    gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
+  postgresql_rhel9_17:
+    repository: '17'
+    description: 'PostgreSQL 17 RHEL 9'
+    osname: 'postgresql'
+    release: 'rhel9'
+    baseurl: 'https://download.postgresql.org/pub/repos/yum/17/redhat/rhel-9-x86_64/'
+    gpgkey: 'https://download.postgresql.org/pub/repos/yum/keys/PGDG-RPM-GPG-KEY-RHEL'
   zfs_dkms_rhel8:
     repository: 'dkms'
     description: 'ZFS DKMS RHEL 8'

From cdf9456456e913510f7da1e2b74a08db229a514c Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Tue, 29 Apr 2025 21:04:45 +1000
Subject: [PATCH 57/89] feat: update psql15 repos for roles (#269)

- update patroni to use packagerepo
- update puppetdb to use packagerepo

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/269
---
 hieradata/roles/infra/puppetdb/sql.yaml | 8 ++++----
 hieradata/roles/infra/sql/patroni.yaml  | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/hieradata/roles/infra/puppetdb/sql.yaml b/hieradata/roles/infra/puppetdb/sql.yaml
index 838300d..baae548 100644
--- a/hieradata/roles/infra/puppetdb/sql.yaml
+++ b/hieradata/roles/infra/puppetdb/sql.yaml
@@ -29,11 +29,11 @@ profiles::yum::global::repos:
     name: postgresql-15
     descr: postgresql-15 repository
     target: /etc/yum.repos.d/postgresql.repo
-    baseurl: https://edgecache.query.consul/postgres/yum/15/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture}
-    gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL
+    baseurl: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/15-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/15-daily/%{facts.os.architecture}/os/PGDG-RPM-GPG-KEY-RHEL
   postgresql-common:
     name: postgresql-common
     descr: postgresql-common repository
     target: /etc/yum.repos.d/postgresql.repo
-    baseurl: https://edgecache.query.consul/postgres/yum/common/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture}
-    gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL
+    baseurl: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/common-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/common-daily/%{facts.os.architecture}/os/PGDG-RPM-GPG-KEY-RHEL
diff --git a/hieradata/roles/infra/sql/patroni.yaml b/hieradata/roles/infra/sql/patroni.yaml
index f925067..f962a62 100644
--- a/hieradata/roles/infra/sql/patroni.yaml
+++ b/hieradata/roles/infra/sql/patroni.yaml
@@ -4,14 +4,14 @@ profiles::yum::global::repos:
     name: postgresql-15
     descr: postgresql-15 repository
     target: /etc/yum.repos.d/postgresql.repo
-    baseurl: https://edgecache.query.consul/postgres/yum/15/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture}
-    gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL
+    baseurl: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/15-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/15-daily/%{facts.os.architecture}/os/PGDG-RPM-GPG-KEY-RHEL
   postgresql-common:
     name: postgresql-common
     descr: postgresql-common repository
     target: /etc/yum.repos.d/postgresql.repo
-    baseurl: https://edgecache.query.consul/postgres/yum/common/redhat/rhel-%{facts.os.release.full}-%{facts.os.architecture}
-    gpgkey: https://edgecache.query.consul/postgres/yum/keys/PGDG-RPM-GPG-KEY-RHEL
+    baseurl: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/common-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/postgresql/rhel%{facts.os.release.major}/common-daily/%{facts.os.architecture}/os/PGDG-RPM-GPG-KEY-RHEL
 
 profiles::sql::patroni::cluster_name: "patroni-%{facts.environment}"
 profiles::sql::patroni::postgres_exporter_enabled: true

From 62f71e1feba5d5144e2f18088933fd3e83c43bc3 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 3 May 2025 01:07:52 +1000
Subject: [PATCH 58/89] chore: change puppetboard python version (#270)

- change python version to follow python3_release fact
- this will follow os-release upgrades

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/270
---
 site/profiles/manifests/puppet/puppetboard.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/profiles/manifests/puppet/puppetboard.pp b/site/profiles/manifests/puppet/puppetboard.pp
index c141e73..121f39b 100644
--- a/site/profiles/manifests/puppet/puppetboard.pp
+++ b/site/profiles/manifests/puppet/puppetboard.pp
@@ -3,7 +3,7 @@
 # This class manages the Puppetboard, a web interface to PuppetDB.
 #
 class profiles::puppet::puppetboard (
-  String  $python_version       = '3.6',
+  String  $python_version       = $facts['python3_release'],
   Boolean $manage_virtualenv    = false,
   Integer $reports_count        = 40,
   Boolean $offline_mode         = true,

From b05acb23f4e1516551984ed887d9b8af0252d654 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 3 May 2025 12:41:23 +1000
Subject: [PATCH 59/89] feat: use custom cert for puppetdb access (#271)

- manually generated certificate using sudo puppetserver ca generate --certname puppetdbapi.query.consul
- saved certificate and private_key in eyaml

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/271
---
 hieradata/roles/infra/puppetdb/api.eyaml      |  2 ++
 .../profiles/manifests/puppet/puppetdb_api.pp | 20 +++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 hieradata/roles/infra/puppetdb/api.eyaml

diff --git a/hieradata/roles/infra/puppetdb/api.eyaml b/hieradata/roles/infra/puppetdb/api.eyaml
new file mode 100644
index 0000000..4bb4232
--- /dev/null
+++ b/hieradata/roles/infra/puppetdb/api.eyaml
@@ -0,0 +1,2 @@
+profiles::puppet::puppetdb_api::public_cert: ENC[PKCS7,MIIJrQYJKoZIhvcNAQcDoIIJnjCCCZoCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEATUl7Aa6W6Q0gWWPet6fufBUtFUO7wCkED8w3NojkDYNR0Cine5+yWupy1FZ0d75mtdjI16DgZ9d2BhNlnbvPrZHuFSfBFj0s6lc0cYs1dpEUwPwPssmfNNfLe+73Fn0e43fguXisBYiE0Xn4x9UqGEIXXnwBqucIo4lkR0QAvhrmgEsNJKrxKV2isBZOnV40hrilnK3fLszGlfEfEuK1ZLrdtQV54Cl/Fpga8OOEk3Ji+WO/qC3WSQ+RWmc+si5L7w6raFLcHb3ZN96BHNVN2h2rBe85RRTg08LT+9Eyge3Fc0/+eoRmzTvnHMc4RptRfvopv5RGGyOma0mExmD6CzCCCG4GCSqGSIb3DQEHATAdBglghkgBZQMEASoEEB2t8I7YdvzPGIuGG1fUIViAgghAKhUoHO1UphPXu/KwYgo3xPSmWWF76B/ZKPd3SdzobYBsfK7CEhRiONU29W/wEzDdn7An2E16bIuecxT114z0i6OqQc3W92jBETWlpV1IRWs4DTb2QWjA4l74KdwRtltpeAKeFjSjuW2+L5UUOgGsaW4lxR0wP/uIqUT76/wQCoCAIw/i+YcIvIJDpeVPKSyaA6GFJbiT8CuG1SzD/1tb+XOxEf3WpeUwVNrPIO1ADjdi3cha6bJDa6Dvrtz8ornYwsfZ9cIlhDb6kmz938+EWpH5swCppfHcMncSd+R1zST6DzhN54+kvWGjvrN79m5+f0al/t3a85iZ6boLXSE8VkPLWZnlAt0ISdCtt0m7luxWWUN3AvmBmLZ5hhjnHUQC4RNCmu3BrjB7bN/nvInZQNcBlArhthiy/DpbdjLpF5kkUq+J+S7I898rG/B8lWrWjYsQvOUM/yiVbpCtsLJS9Pv1UjlkHcere6YgOq4gZKaESF19npV6SLU2MfC+Raefj1biE+haOOjpDdR+xQLAHmZqgOBUFMkYh1RH77zg2QPtz5aDLGypO/yVuJDcuSGV6qpoxc0uu7EmihfOg2cHF6FtSlStwFQYw3mG3oyuByv527lVRUjNHOx85bXeFccb96lyTzStAopLADo9nuOHjDxs6qzXj4h0y5w0ODh3Wyy/h4EXYaTrXuSB/FJJb1rvToC+XJ7ABxzt0rB8ySNtt+DFRssZQ5ZXWF6T88YKLcigKYGNTGmf92Iwpq6+hw0NEF1OWy7aHDzog6xORilgy6zcPTWkz2TUxzOuwN6Y0UeLlj+C4r+hl17/9aYhls6UJ+5xF+ZNcJmqEXqZ5HHykcYRwaWI0FF4tkbsto8Is2/aZVfeQc/2JZ+9IbLXlh1Km6hJxWJmw/S9RwTXVK7kGO/IlIoQiYTFYoeSU4RDPVUXZTBmGxuBmz37JPVMLXkL6tGUPwTz6pa+AMppT/qMLC8y2LhLm+eRsfz4w2ySc/kBR0FKsD0Z1h9h4zM+VtNnxaSYmxkFG77pX+bi/ToQqaydWblf0NPdw2t+uoULzkxxhX3wjZi8V9gGOhZ7s7YUKJFljZZYcl0MnDab+xGjD8DGiq/vHqTLXm8DYpVOxsryGIJ3zXf5KPvo8y6/wQAkKq6Vb4lraqkg9m5wGLxQDemE4h2OWgjcnWOXx/N9bcVO0xMyqrFo337wPoZ+hYQhwxzrfiQZ87nLe28OstaWS8OK6KoAx95LvMypaFURf+EoZkYO8wFiLmFBNAMMOkf/TJjmXoeDw46Qv1sZi57239pgzxz6RXEjd5TBURli7tSaniqKNarRY7ZoYymzYBv9Kyj6zQGgXxozhQsMsju9fTo2l+bXQ2siBljHnNkI+I3aO5Q6FLpM57h5xhA86ayJzfaKSbniMARGY2inG3qUfKafgQUvrYBxaIxSBAg7LfE2GRJx8gEioATFZpclrm+0fP3xaXW3I/wiyl/EPKIP2aPP8lqJam/KWXZ46nYdgrKIrg51tXp/YUcgR6geZYHIBWkAgofJeThKPz9ervLzu3dYS0FgMiRcyCOXfI4nttW/QCNl5a19UXXYpSgj0MOAKuSZYkHYgaSt7DNt3sZtWLtCZ1M/QFLSiqsfUAULVwUJpOCS4Ul0Bn9gu038xCBCkaQ8VnSHtvl6NsCUHDhk/JGq5pmZjrE5zTEnlMBUBoQ1sun/HwLAnoX24KV+3pzwul0eCLm2pBndWvgnHsEY7COiookx7mwvg93xuejN7zQk/NAJp4L3haT5ueVTcUcEsTPmsIDMn2xg2HSGLum6yG02XPMBYJlG/GHtu2kuvOV2UFqxkzje4FB3cNishelQ1VRDOBJodt8xmfKkgPciFChEOVe5OY7AbBvKIBab+kjbG78guGReqkmePFkEtsnL7KouiERojAVsGXtvqOT2dQvO7xLrozLk+kY/Xk6HkGedmc2PUEc/CSKPy73k2a154ByzwfOaAaTM1XCvo4Ff7hTA7VHUu3rWpHmd2LZbKN1nlGbrrX0Wk0jt72OsRWRzgKp+81jEkNh/hbD9xCjmIbzdloOmcJJbcyikmThVpFaUMaHowZmrBtQxE7pR2ARbhVvNXH3fZQnIpxMcHEPoKh12pOTlp+GIO8H1EsGZ6tOjXniBiy/szoa8Oi/eJp/Co8uRoDSyBc5t6ZD6ciLHOVG4c0nCdMHaouA/EXNe/EOzLg4fYk7cLBNGoaBUo8LbXv0Z2tkhtE1ctl8NBvZS3X80VchtmQg7lVlqZUEkcXEtoadrjRpiWL9EW68mzjTsejePMNBb6CCL4zGqQwCfA1zUVtlNJslguClQ2u0IlqPjBYsj3Xy+leg24YrHKB1zEu1/aGuVxCBlJMozQj//5OCTp+1iO0EExnGBYjZuk8UTYrUj9FZeu5GiRlBvky1HBE/fq4LPD7l/Gr0npAwKJofIglA2DZe9Sr4VmA8oi4vsmbpmtyPLVa/pfVXrl/w2dLH/Y2TI8MbPFMvjtAgdlK6endrxpb9EC2YeWrFibDXL9EsOAXo5droa6WyIDoVr7GBZ0Faa19uW2IZf0fw02tz0L1Bg/4RopeIZpbH0CKCwpqg5GNb7gKvKkXt9ugI84ZGnF5CgrqXH+lXXtMgHsEkUQ6vAJeKioLSMVla6Pu/BdztDKBVuKTEzV/lH/nbR2qhjlIEm+AntndtRNU3J6Aakje0keGjDV66paCnh/v7fha3SOPkgCV8OxrqMDAUl9/RxB907OF/Ethg4F/gsWfwDJLcAoS206rV6r+VZyurb3xx6HSLFzh+MMBgNlJhf]
+profiles::puppet::puppetdb_api::private_cert: ENC[PKCS7,MIIOHQYJKoZIhvcNAQcDoIIODjCCDgoCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAR7TO46Sg7DTeJmy+YMKpcKgcbbyDed4zHib7Jii2ltlFBMzFAJ49u986i13pSE0qaPIgydc94ILKc0oTwTWvstbiBJB2NYikswKHIJwKKSTzzbfCz1/eyfVKwKyp3DakdImdPClmjZIp/eQyWsLkZjxmJ5T9MVHSo3l8sNVY3WyGbKwEi0TwZabE9Op8cvFTifcBL5zDwKMrhwghAPEoVz8DALoMOko8Lp2K7yCo+LvyvY/Ib9CtIfbgpXQTo2NDbww29/KWHqw6PjzRdyuswUDdnCuLOaThUQkDopTZyhhJ9Y5z5/7Doo/zGYQLBkyThjtKkKwsPlq1eGkjFzY/ITCCDN4GCSqGSIb3DQEHATAdBglghkgBZQMEASoEEAvibroua3O2ot73run7uWKAggywIm7gZ/o/vc47kaCICTS2LtsHDmTe9HHqujdAdQLlBGvEhasKGDMr8s54aIbNUbLhD82Vuvwjl2GShqSg/IATfwzDPoqm982KrVZd/9kO61o3vBdtqZO1+InbY7iDbTqf45A7Ar3545aF+/y30WDamFyECwCePrQu1Y9bw6VhLTEVYO+A8zr8y0CYRKAtWNYcPpTP/3VO8/RKtMV+nWrUzS00zAp56lA7xeOzdeKdHzCkpKeaHrpSEN6YGrZmAhTdKBxHYiez9h3eZprEtqoAz0I5DL9Y54t74+pZxicFjQKYPxUGccqg0dIkFZUgE1dxuOkASspydJ890xC/SYJZxREeuF9SPCUkT4Szc814/JHzfqrHt5ZfdzQcBX0xjtEfwoud+Hc4MVerT4cmhyaJLP1/jEe8l3B3Cj4uVilbfAyxOAmAHZl4lcKawdzqUm1os6bsBzCwQU0usKYlW1vczvjvxHvq5z/+IQvCvvlPlUrVIlB7XaiIihkW4LbgZBKNaCWW8rihewX5FUTtvjveHzkGqXnzBBF8WfQmNvVtjKFGdkz3teOHShZOZO7fYo4rHGAV+KVz3kzSCZ8pgqAUsYYW5z/aViNnFuNdDz8xiq05DfUtD3GwbydH2mr5IN8bAFwufAidKEfuQeAusCuS5k6PRJyXK81ErqYMW/H+mSLVw9jCNOqqGGfiYdfcx8oNw+U1Vz/fDmQIQ+eT/JKDSOCbBYjrydC9WBzvgRngDoLQ/UXGCnIENqLJekvVJ0x9KTUhrzQyZVky+INyzp3i5UDyu2uLH55vJEI7qSmcM0Y8Eyj3Buc+0k739OQCC56IiQm/Aw6Uh+6Vi+/MfevcI5K0etMgTZ+dIhRAJAT5Jd4vRu4ge8M1HqEOQQj5lQfO+WtHcgwFyg1dGeQF6I6iy0JfMqXZVDsEkyCsDN6ZDaLcgMQN0FYWSsqy7tCzU9ZbvQ/pAHT/Ed1mNDWs+w0YzgcVv8+XCBGVzunpqcjbi08aKDYIKOIL5EBVOfDwGIeYjBuJhINQa49tNryXEkjZs+OxXDM4YvvksvwfNq4P4OLNJHoQ4DCEyqVbdqGGksRjNtU28Lu8y2M5eX6ehG8ismBMSgUy0SIbR5vTDVNjguo4HD7slVNYqERfMjlwHbnOLajoPl8nWIrhW3glWKBvJ1bQKWU81TRawlCAQ3k5DFnebFgoBagtqo0+Zoygt29Bz38wwqfF90jvfzVEM0sy4X+7FxZFYX2WTWKyZV3+16a7hWBjMbJeulU1chzs1V0/lPMXVEa+c3xcO8W2DyrX4wR27fUEfjPmug49tMhMSFOZXEOwzUMsXjz3K4XhEcC0mtYHDMT0EYtRX8pHe62RkMILuJGRHvw3Wwg2GzZRMpZfPQy8jLhkOhUSGc7z4XEWC/PdLmfGjBTZsZ6G2FfKlGLbSxOlgQlew2sZ+oNTHRrN+aMrkgyG11TheXDbE0y+Sxls+13DEYbMaflq5wjE2FlvD3b5X7/Pu4BizQI7Ski7jjb5qcqik8xoin2qGfhnY6H6PNSPz67J1RTxVZjHgUtZQKG56e+aKIv5FFyjFTgEPRqj3Yyrzh7+E6gt+LETh3vuJDUxKPLZskLD03xP6E/yiLbIu5DIYzK9+uuS/JGF3OibE93hgwp9uzDFirDzRbcuDVDxZyToSfH8P7m58lhp/FiKO3rpUqRqaC/GXz5xbekQ+yGuMKk6qEmiPj7B3pHm2Ic2tJIGTpTNCDNfsqefplUJXbKskiKRxOP04yCIn3XuHu8jUDhWZW848RWHgXfySdqEAFKrvsyamJKXz/1IGIdDzOgnAcd/wiudhZ41gDaCyA1DQYxU9T04106eD9aTZwj+mS9uZmfwHfROiBinC1ZAHsXfjLtwVZj3gpdUsieIqAiuAuMKle9yOMn3np4NEu5vjj/SOZeVWCvBfcPg8VB+uj6s7FtXRH5jdxkaB72PykYa8aTAbNKqcw/qlowv0k/vWnWa++fFFVfbg+VZ3a6oEW2YKvcVefQg/R4nEeTusmH/tnOl7SHLlJ4WLQ+U+v0GLeCY6zJKS6WweN+6SXIbkB9BLES2VpvxlYAmjqLlQuJBpcIuxL1SwCIkKUXdWjQqp6WzfS4XSxuxizZNL8xlEjXSJzFXvo+rPZXF37gHvmLHnjJkQdhctZUjyzDUfgLdR4FcMsp/U8qugWtbbemARFtrXHT9D3M/hVp0z/0ho80LB2chI1nOfY0u8GXgHtJfwZtoeyctud6JLo4zUxZy0zQMqIqgpytVNhjMPOIK5sWpKCKscPlg8+vANhPQHze7Wc8PXwC7jbSMH4QqqnGz+fPB/KKl+wz4jICe/vu7fNJRq0pehibPifXJtBDBH+i+s/U3ZX2WECKPYsF0DXq8Askisa3b97WPO7oWk8ghzE8ZXZs5CypkLpq+oPRg/WrsRJea+EWmkzKmBXtS23AQzhItNB0KFP18elLVmb6atYNs4lzv22ptifD+5wbYmk+/mAVgInqqixLPAy2uofCXvlXhIPd4vgkwpjar/lxcZWGtX7c7EUzDND1AyirMjcZyTaMsHEzYybT3S/8n8OgxVvyemLvgHnmZxmWnUBFUhItQ2Pt6iF2Dn68huo+0PE5K5qlKzoRPbfNLrNYImn4KxmLW5+bT2STe1j8aowyY3ROY1r+4BxHIbDknCb5aj6N4WR14SeUWSSPu23oK1VDLBlQUMiHHqfJwu0NiGhsI8Vl0Mp5Slp6FK0UvZz2O7C+x0HGaWVK9mMuP40BKHjEo7T97ZoXMN0uX4GmBQ3jEwXzsPpuPbZI5JnwwEyp5YX+IyYWJyCww2fntSQ/fS509b6mvG5isUMK3lpiogucROtJTA1KrODrocpTgUGvIwDUP/tosGT57lmXjTmHbTf6WjCHIUZsrbT5ycjYNJI0sn+dfx3Tba289CEeBEaX2FefnHX+o8HMLlRI4JmiD6faswCjC2IFMjDVfkJoie45Y+8Q9LSVZ8TkZey1QucNRrMIKpNLo4cPDCdUJIOPUDbfX4I1g0Qhk7T5WYX/JUtiaUgKiR5Gc/PHmV8oL8G9P+OSgg2Lc5XaN7/PevHCiBOLwAXsKbSCwL7oGmxyw90+sHIZr+ZyClNnfuXE3UrW7NXQI6mCxvQWKtJKn1trO17I6Cd00RMfC3uPy5/LIicubunfvDjp3BlaatxIrzDg3rVnFPX0dKzt/s4iuSyK3IT0le/x9OBLIp4EhdQdvd2HOFYxOtjxzbvWRzWrqGpBsLQLg7EG5adrqg69VOQMCT2pq0sCJp6hk4yGHbuz4hoX2PIeiI3l/Z0TwDGopGWW3uWRWxH/6bq1a6idgxHYhruUmR8aFpUKSsVyodiDV1PtK1zj5mKcZdeXx0THBYTEUWyo3RI9iLUnMcDYyPRTjCKoCm6pPS7yjIT7tq+MUi3iWo2XijY8AaV/K+WrnkcqoQH2cXsrBoH0sZMksdWwYBYz97+RKkx3FSxgdXJb1xxr0X21oPW1PhJq/zHSxe3OmxwYMPeZsGPaG8UngxTdfTQ3KPn+mN9NRXvlXkqudGJa/d/QMinCf/tp9eU1WX5xktqDgqBbPD391+6gcDsFgdidFdpLKf6yGLEovTwE9GrLX7srAC2dPOxviWekqng8H0N9RjHT06NVD9kAnnR/01ZkmmPzdP7vLX9imy0j/huTpN8i9XpFS6za4jdGt46Y5SfRKg/4qpwtE6meElgiOnTkP2uBI5cF4czaK1rYr1ZlwoneFzNHnBVFctAP7CD+w98hYexQ5RrdUYgFrMB5+88Y1IrI8Gd02RCEycPCoxdEFXqBVWmu2N/fuovgihTn9qf/mUxpQPsiUHjCgmV5w+ALs02oiRf08b8mTBAjy8utziFA6efpAppv98bQUfTs8lyNn7sH6OCoa4ZKeB51wqPXZBsKZ8Ja9VDZXjpCV+MB0b9XuSgOx2Y3UD/GJ/+eW7J6MiYFTHrhB1kRert5vRdF4M2KMoyYnVRFMBRqimnNBq6xrOvz4D6dSiquhFRtxlnb6dPCyRgHImsFbR/qdDukLB6DB+vfjzZQArLbkaMVF86hX1+9xP+BdZPsnVapY9u+DSZP7ZkEP/VftTl7uJZnqAWUis+jOncUizmzSIpV6cZO7SVdiQicHiRm6VSOmpBhniRKYN13Ct8B0g+JJeyMlx6OGALMLvcXKDXxJNvyYL9FuXyRvOygButRRjvLsEku+fl9ESD27r56ZNExf7w411cGbKQ+obhSnMsBM2qGBDR97VJyLGhbFX7SKl7CGOAWZF3TbOe4PU4Ty75qb8+U5R8amQ1FFQD4r65M=]
diff --git a/site/profiles/manifests/puppet/puppetdb_api.pp b/site/profiles/manifests/puppet/puppetdb_api.pp
index bdf7532..03567b3 100644
--- a/site/profiles/manifests/puppet/puppetdb_api.pp
+++ b/site/profiles/manifests/puppet/puppetdb_api.pp
@@ -1,5 +1,7 @@
 # configure the puppetdb api service
 class profiles::puppet::puppetdb_api (
+  String $private_cert,
+  String $public_cert,
   String $postgres_host = lookup('puppetdbsql'),
   String $listen_address = $facts['networking']['ip'],
   Stdlib::Absolutepath $java_bin = '/usr/bin/java',
@@ -24,6 +26,24 @@ class profiles::puppet::puppetdb_api (
 
     contain ::puppetdb::server
 
+    file { '/etc/puppetlabs/puppetdb/ssl/private.pem':
+      ensure  => 'file',
+      content => Sensitive($private_cert),
+      owner   => 'puppetdb',
+      group   => 'puppetdb',
+      mode    => '0600',
+      notify  => Service['puppetdb'],
+    }
+
+    file { '/etc/puppetlabs/puppetdb/ssl/public.pem':
+      ensure  => 'file',
+      content => $public_cert,
+      owner   => 'puppetdb',
+      group   => 'puppetdb',
+      mode    => '0600',
+      notify  => Service['puppetdb'],
+    }
+
     # generate the minute for the cron job using fqdn_rand
     $random_minute = fqdn_rand(60)
 

From 1b8f50786f14d24fc1343c397af9c532b4f2177d Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 3 May 2025 22:25:10 +1000
Subject: [PATCH 60/89] feat: ensure the vault audit_log exists (#272)

- without this, vault will not take a leadership role

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/272
---
 site/profiles/manifests/vault/server.pp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/site/profiles/manifests/vault/server.pp b/site/profiles/manifests/vault/server.pp
index 84398f4..d73a4a3 100644
--- a/site/profiles/manifests/vault/server.pp
+++ b/site/profiles/manifests/vault/server.pp
@@ -15,6 +15,7 @@ class profiles::vault::server (
   Stdlib::Absolutepath $ssl_crt     = '/etc/pki/tls/vault/certificate.crt',
   Stdlib::Absolutepath $ssl_key     = '/etc/pki/tls/vault/private.key',
   Stdlib::Absolutepath $ssl_ca      = '/etc/pki/tls/certs/ca-bundle.crt',
+  Stdlib::Absolutepath $audit_log   = '/var/log/vault_audit.log',
 ){
 
   # set a datacentre/cluster name
@@ -85,6 +86,14 @@ class profiles::vault::server (
       ]
     }
 
+    # ensure the vault audit log exists
+    file { $audit_log:
+      ensure => 'file',
+      owner  => 'vault',
+      group  => 'vault',
+      mode   => '0600',
+    }
+
     service { 'vault':
       ensure    => true,
       enable    => true,

From 3079f7d0001f5a693dae877f26e0bf8b4ed9d69b Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 3 May 2025 23:51:17 +1000
Subject: [PATCH 61/89] feat: enable use of dhcp addresses in networkd (#273)

- change ipaddress to be optional
- add dhcp option

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/273
---
 modules/networking/manifests/static.pp            | 3 ++-
 modules/networking/templates/networkd-network.erb | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/networking/manifests/static.pp b/modules/networking/manifests/static.pp
index 018ef89..8110d8a 100644
--- a/modules/networking/manifests/static.pp
+++ b/modules/networking/manifests/static.pp
@@ -1,10 +1,11 @@
 # manage static interfaces
 define networking::static (
   String                $type,
-  Stdlib::IP::Address   $ipaddress,
   Stdlib::IP::Address   $netmask  = '255.255.255.0',
   Integer[100-9200]     $mtu      = 1500,
+  Boolean               $dhcp     = false,
   Optional[Boolean]     $forwarding = false,
+  Optional[Stdlib::IP::Address]         $ipaddress = undef,
   Optional[Stdlib::IP::Address]         $gateway = undef,
   Optional[Array[Stdlib::IP::Address]]  $dns = undef,
   Optional[Array[Stdlib::Fqdn]]         $domains = undef,
diff --git a/modules/networking/templates/networkd-network.erb b/modules/networking/templates/networkd-network.erb
index 298304d..b2ffc1e 100644
--- a/modules/networking/templates/networkd-network.erb
+++ b/modules/networking/templates/networkd-network.erb
@@ -2,6 +2,9 @@
 Name=<%= @title %>
 
 [Network]
+<% if @dhcp == true -%>
+DHCP=yes
+<% else -%>
 <% if @ipaddress && @netmask -%>
 Address=<%= @ipaddress %>/<%= IPAddr.new(@netmask).to_i.to_s(2).count('1') %>
 <% end -%>
@@ -14,6 +17,7 @@ DNS=<%= Array(@dns).join(' ') %>
 <% if @domains -%>
 Domains=<%= Array(@domains).join(' ') %>
 <% end -%>
+<% end -%>
 <% if @bridge and @bridge != true -%>
 Bridge=<%= @bridge %>
 <% end -%>

From a70b6492b07260ecb9c45a06c8e19c83e0197721 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 3 May 2025 23:51:29 +1000
Subject: [PATCH 62/89] feat: update consul/dnsmasq (#274)

- update params with bind/advertise addr
- update params with anycast ip option
- migrate dnsmasq config to template

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/274
---
 site/profiles/manifests/consul/server.pp        | 9 ++++++---
 site/profiles/templates/consul/dnsmasq.conf.erb | 6 ++++++
 2 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 site/profiles/templates/consul/dnsmasq.conf.erb

diff --git a/site/profiles/manifests/consul/server.pp b/site/profiles/manifests/consul/server.pp
index f71c567..9a57ae7 100644
--- a/site/profiles/manifests/consul/server.pp
+++ b/site/profiles/manifests/consul/server.pp
@@ -45,6 +45,9 @@ class profiles::consul::server (
   Boolean   $disable_update_check   = true,
   Boolean   $join_remote_regions    = false,
   Array[String] $remote_regions     = [],
+  Stdlib::IP::Address $bind_addr            =  $facts['networking']['ip'],
+  Stdlib::IP::Address $advertise_addr       =  $facts['networking']['ip'],
+  Optional[Stdlib::IP::Address] $anycast_ip =  undef,
 ) {
 
   # wait for all attributes to be ready
@@ -112,8 +115,8 @@ class profiles::consul::server (
           'ui'                   => $enable_ui,
           'ui_config'            => { 'enabled' => $enable_ui_config },
           'performance'          => { 'raft_multiplier' => $raft_multiplier },
-          'bind_addr'            => $::facts['networking']['ip'],
-          'advertise_addr'       => $::facts['networking']['ip'],
+          'bind_addr'            => $bind_addr,
+          'advertise_addr'       => $advertise_addr,
           'retry_join'           => $servers_array,
           'retry_join_wan'       => $remote_servers_array,
         },
@@ -143,7 +146,7 @@ class profiles::consul::server (
       owner   => 'root',
       group   => 'root',
       mode    => '0644',
-      content => "server=/${domain}/${::facts['networking']['ip']}#${dns_port}\n",
+      content => template('profiles/consul/dnsmasq.conf.erb'),
       require => Package['dnsmasq'],
       notify  => Service['dnsmasq'],
     }
diff --git a/site/profiles/templates/consul/dnsmasq.conf.erb b/site/profiles/templates/consul/dnsmasq.conf.erb
new file mode 100644
index 0000000..57ecef6
--- /dev/null
+++ b/site/profiles/templates/consul/dnsmasq.conf.erb
@@ -0,0 +1,6 @@
+server=/<%= @domain %>/<%= @bind_addr %>#<%= @dns_port %>
+<% if @anycast_ip -%>
+listen-addr=<%= @anycast_ip %>
+<% else -%>
+listen-addr=<%= @bind_addr %>
+<% end -%>

From ed947dee595c463993fda40fb072cef533b647c8 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 4 May 2025 00:07:45 +1000
Subject: [PATCH 63/89] fix: listen-addr -> listen-address (#275)

- listen-address is the correct option

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/275
---
 site/profiles/templates/consul/dnsmasq.conf.erb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/site/profiles/templates/consul/dnsmasq.conf.erb b/site/profiles/templates/consul/dnsmasq.conf.erb
index 57ecef6..1e0a328 100644
--- a/site/profiles/templates/consul/dnsmasq.conf.erb
+++ b/site/profiles/templates/consul/dnsmasq.conf.erb
@@ -1,6 +1,6 @@
 server=/<%= @domain %>/<%= @bind_addr %>#<%= @dns_port %>
 <% if @anycast_ip -%>
-listen-addr=<%= @anycast_ip %>
+listen-address=<%= @anycast_ip %>
 <% else -%>
-listen-addr=<%= @bind_addr %>
+listen-address=<%= @bind_addr %>
 <% end -%>

From f322440d010ca82d31fb56a7c5fe22eccedae19f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 9 May 2025 22:07:42 +1000
Subject: [PATCH 64/89] feat: setup anycast consul dns (#276)

- manage frrouting repo/ospf
- change to systemd-networkd
- enable ospf on incus nodes bridges

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/276
---
 .../nodes/ausyd1nxvm2005.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2006.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2007.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2008.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2009.main.unkin.net.yaml  | 47 +++++++++++++++++++
 hieradata/roles/infra/incus/node.yaml         | 10 +++-
 6 files changed, 243 insertions(+), 2 deletions(-)
 create mode 100644 hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml

diff --git a/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb4494
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+profiles::consul::server::anycast_ip: 198.18.19.14
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb4494
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+profiles::consul::server::anycast_ip: 198.18.19.14
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb4494
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+profiles::consul::server::anycast_ip: 198.18.19.14
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb4494
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+profiles::consul::server::anycast_ip: 198.18.19.14
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml
new file mode 100644
index 0000000..fbb4494
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+profiles::consul::server::anycast_ip: 198.18.19.14
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index a5cadbc..9b761a7 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -26,12 +26,12 @@ consul::services:
       - 'incus'
       - 'container'
       - 'lxd'
-    address: "%{facts.networking.ip}"
+    address: "%{hiera('networking_loopback0_ip')}"
     port: 8443
     checks:
       - id: 'incus_https_check'
         name: 'incus HTTPS Check'
-        http: "https://%{facts.networking.fqdn}:8443"
+        http: "https://%{hiera('networking_loopback0_ip')}:8443"
         method: 'GET'
         tls_skip_verify: true
         interval: '10s'
@@ -110,6 +110,12 @@ frrouting::ospfd_interfaces:
     area: 0.0.0.0
   loopback2:
     area: 0.0.0.0
+  brcom1:
+    area: 0.0.0.0
+  brdmz1:
+    area: 0.0.0.0
+  brwan1:
+    area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
 

From 537a2077790037776a440baee264534207bca1da Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 9 May 2025 22:10:35 +1000
Subject: [PATCH 65/89] feat: update upstream ip for consul dns (#277)

- set bind resolvers to use consuls anycast address for forwarding

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/277
---
 hieradata/country/au/region/syd1/infra/dns/resolver.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
index b26491e..d6d8dca 100644
--- a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
@@ -3,6 +3,4 @@ profiles_dns_upstream_forwarder_unkin:
   - 198.18.13.14
   - 198.18.13.15
 profiles_dns_upstream_forwarder_consul:
-  - 198.18.13.19
-  - 198.18.13.20
-  - 198.18.13.21
+  - 198.18.19.14

From 51d6c1e81df321556186f610242f10789ae5f34e Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 10 May 2025 06:57:05 +1000
Subject: [PATCH 66/89] fix: enable dns resolver access for dmz1 (#278)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/278
---
 hieradata/roles/infra/dns/resolver.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hieradata/roles/infra/dns/resolver.yaml b/hieradata/roles/infra/dns/resolver.yaml
index f94eb93..e9adbdf 100644
--- a/hieradata/roles/infra/dns/resolver.yaml
+++ b/hieradata/roles/infra/dns/resolver.yaml
@@ -201,3 +201,4 @@ profiles::dns::resolver::views:
       - acl-main.unkin.net
       - acl-nomad-jobs
       - acl-common
+      - acl-dmz

From bb6f6cbd49309546f68b957fd02800bf2a4e0167 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 10 May 2025 23:00:03 +1000
Subject: [PATCH 67/89] feat: anycast dnsmasters (#279)

- change dns masters on incus to anycast for bind
- change to networkd to support anycast/loopbacks

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/279
---
 .../au/region/syd1/infra/dns/resolver.yaml    |  3 +-
 .../nodes/ausyd1nxvm2029.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2030.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2031.main.unkin.net.yaml  | 47 +++++++++++++++++++
 4 files changed, 142 insertions(+), 2 deletions(-)
 create mode 100644 hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml

diff --git a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
index d6d8dca..740336c 100644
--- a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
@@ -1,6 +1,5 @@
 ---
 profiles_dns_upstream_forwarder_unkin:
-  - 198.18.13.14
-  - 198.18.13.15
+  - 198.18.19.15
 profiles_dns_upstream_forwarder_consul:
   - 198.18.19.14
diff --git a/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
new file mode 100644
index 0000000..00d319e
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_master_anycast_ip: 198.18.19.15
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_master_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
new file mode 100644
index 0000000..00d319e
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_master_anycast_ip: 198.18.19.15
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_master_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml
new file mode 100644
index 0000000..00d319e
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_master_anycast_ip: 198.18.19.15
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_master_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent

From 3e0141bb1b5bfa11c11113b97a00328abe0d9fbc Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 11 May 2025 11:39:00 +1000
Subject: [PATCH 68/89] feat: change to anycast resolver (#280)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/280
---
 hieradata/common.yaml                         |  6 ++-
 .../nodes/ausyd1nxvm2032.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2033.main.unkin.net.yaml  | 47 +++++++++++++++++++
 .../nodes/ausyd1nxvm2034.main.unkin.net.yaml  | 47 +++++++++++++++++++
 site/profiles/manifests/dns/base.pp           |  6 +--
 5 files changed, 148 insertions(+), 5 deletions(-)
 create mode 100644 hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index ae0a661..3bee9e1 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -182,9 +182,11 @@ vault::manage_service_file: true
 vault::manage_config_dir: true
 vault::disable_mlock: false
 
+profiles::dns::base::nameservers:
+  - 198.18.19.16
 profiles::dns::master::basedir: '/var/named/sources'
-profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
-profiles::dns::base::use_ns: 'region'
+#profiles::dns::base::ns_role: 'roles::infra::dns::resolver'
+#profiles::dns::base::use_ns: 'region'
 profiles::consul::server::members_role: roles::infra::storage::consul
 profiles::consul::token::node_editor::accessor_id: '024e27bd-c5bb-41e7-a578-b766509e11bc'
 profiles::consul::client::members_lookup: true
diff --git a/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
new file mode 100644
index 0000000..92f6c57
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_resolver_anycast_ip: 198.18.19.16
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
new file mode 100644
index 0000000..92f6c57
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_resolver_anycast_ip: 198.18.19.16
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml
new file mode 100644
index 0000000..92f6c57
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml
@@ -0,0 +1,47 @@
+---
+hiera_include:
+  - frrouting
+
+# networking
+dns_resolver_anycast_ip: 198.18.19.16
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp
index 12d2d99..5542515 100644
--- a/site/profiles/manifests/dns/base.pp
+++ b/site/profiles/manifests/dns/base.pp
@@ -1,13 +1,13 @@
 # profiles::dns::base
 class profiles::dns::base (
-  String $ns_role = undef,
   Array  $search  = [],
   Array  $nameservers = ['198.18.13.12', '198.18.13.13'],
-  Enum[
+  Optional[Enum[
     'all',
     'region',
     'country'
-  ]      $use_ns  = 'all',
+  ]]      $use_ns           = undef,
+  Optional[String] $ns_role = undef,
 ){
 
   # install bind_utils

From 87a6c73578363157a1298b3d9bda37116a4833ab Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 11 May 2025 16:36:04 +1000
Subject: [PATCH 69/89] neoloc/loopback_dns (#281)

- manage all interfaces in dns (except lo and anycast)
- move loopback0 anycast addresses to be anycast0

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/281
---
 .../nodes/ausyd1nxvm2005.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2006.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2007.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2008.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2009.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2029.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2030.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2031.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2032.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2033.main.unkin.net.yaml  |  4 +--
 .../nodes/ausyd1nxvm2034.main.unkin.net.yaml  |  4 +--
 hieradata/roles/infra/incus/node.yaml         | 12 ++++++++-
 site/profiles/manifests/dns/base.pp           | 21 ++++++++++++++-
 site/profiles/manifests/dns/client.pp         | 27 ++++++++++---------
 14 files changed, 67 insertions(+), 37 deletions(-)

diff --git a/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
index fbb4494..f873956 100644
--- a/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2005.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
index fbb4494..f873956 100644
--- a/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2006.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
index fbb4494..f873956 100644
--- a/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2007.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
index fbb4494..f873956 100644
--- a/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2008.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml
index fbb4494..f873956 100644
--- a/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2009.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('profiles::consul::server::anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
index 00d319e..ad02274 100644
--- a/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2029.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_master_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
index 00d319e..ad02274 100644
--- a/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2030.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_master_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml
index 00d319e..ad02274 100644
--- a/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2031.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_master_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
index 92f6c57..69fc05d 100644
--- a/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2032.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
index 92f6c57..69fc05d 100644
--- a/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2033.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml
index 92f6c57..69fc05d 100644
--- a/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml
+++ b/hieradata/nodes/ausyd1nxvm2034.main.unkin.net.yaml
@@ -11,7 +11,7 @@ networking::interfaces:
     type: physical
     forwarding: true
     dhcp: true
-  loopback0:
+  anycast0:
     type: dummy
     ipaddress: "%{hiera('dns_resolver_anycast_ip')}"
     netmask: 255.255.255.255
@@ -24,7 +24,7 @@ frrouting::ospfd_redistribute:
 frrouting::ospfd_interfaces:
   eth0:
     area: 0.0.0.0
-  loopback0:
+  anycast0:
     area: 0.0.0.0
 frrouting::daemons:
   ospfd: true
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 9b761a7..1fbe7ba 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -13,10 +13,18 @@ profiles::pki::vault::alt_names:
   - incus.query.consul
   - "incus.service.%{facts.country}-%{facts.region}.consul"
 
+profiles::pki::vault::ip_sans:
+  - "%{hiera('networking_loopback0_ip')}"
+  - "%{hiera('networking_loopback1_ip')}"
+  - "%{hiera('networking_loopback2_ip')}"
+
 profiles::ssh::sign::principals:
   - incus.service.consul
   - incus.query.consul
   - "incus.service.%{facts.country}-%{facts.region}.consul"
+  - "%{hiera('networking_loopback0_ip')}"
+  - "%{hiera('networking_loopback1_ip')}"
+  - "%{hiera('networking_loopback2_ip')}"
 
 # configure consul service
 consul::services:
@@ -65,10 +73,12 @@ profiles::yum::global::repos:
     gpgkey: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-openzfs-2022
     mirrorlist: absent
 
+# dns
+profiles::dns::base::primary_interface: loopback0
+
 # networking
 systemd::manage_networkd: true
 systemd::manage_all_network_files: true
-#networking::use_networkd: true
 networking::interfaces:
   enp2s0:
     type: physical
diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp
index 5542515..8d10a85 100644
--- a/site/profiles/manifests/dns/base.pp
+++ b/site/profiles/manifests/dns/base.pp
@@ -7,6 +7,7 @@ class profiles::dns::base (
     'region',
     'country'
   ]]      $use_ns           = undef,
+  String $primary_interface = $facts['networking']['primary'],
   Optional[String] $ns_role = undef,
 ){
 
@@ -43,6 +44,24 @@ class profiles::dns::base (
   }
 
   # export dns records for client
-  profiles::dns::client {"${facts['networking']['fqdn']}-default":}
+  $facts['networking']['interfaces'].each | $interface, $data | {
 
+    # exclude those without ipv4 address, lo and anycast addresses
+    if $data['ip'] and $interface != 'lo' and $interface !~ /^anycast[0-9]$/ {
+
+      # use defaults for the primary_interface
+      if $interface == $primary_interface {
+        profiles::dns::client {"${facts['networking']['fqdn']}-${interface}":
+          interface => $interface,
+        }
+
+      # update secondary interfaces
+      }else{
+        profiles::dns::client {"${facts['networking']['fqdn']}-${interface}":
+          interface => $interface,
+          hostname  => "${facts['networking']['hostname']}-${interface}",
+        }
+      }
+    }
+  }
 }
diff --git a/site/profiles/manifests/dns/client.pp b/site/profiles/manifests/dns/client.pp
index 3dca748..9e2d637 100644
--- a/site/profiles/manifests/dns/client.pp
+++ b/site/profiles/manifests/dns/client.pp
@@ -1,30 +1,31 @@
 # profiles::dns::client
 define profiles::dns::client (
-  Boolean   $forward    = true,
-  Boolean   $reverse    = true,
-  Integer   $order      = 10,
+  Boolean   $forward      = true,
+  Boolean   $reverse      = true,
+  Integer   $order        = 10,
+  String    $interface    = $facts['networking']['primary'],
+  Stdlib::Fqdn $hostname  = $facts['networking']['hostname'],
+  Stdlib::Fqdn $domain    = $facts['networking']['domain'],
 ){
 
-  $intf = $facts['networking']['primary']
-  $fqdn = $facts['networking']['fqdn']
-  $last_octet = regsubst($::facts['networking']['ip'], '^.*\.', '')
+  $last_octet = regsubst($facts['networking']['interfaces'][$interface]['ip'], '^.*\.', '')
 
   if $forward {
-    profiles::dns::record { "${fqdn}_${intf}_A":
-      value  => $::facts['networking']['ip'],
+    profiles::dns::record { "${title}_A":
+      value  => $facts['networking']['interfaces'][$interface]['ip'],
       type   => 'A',
-      record => $::facts['networking']['hostname'],
-      zone   => $::facts['networking']['domain'],
+      record => $hostname,
+      zone   => $domain,
       order  => $order,
     }
   }
 
   if $reverse {
-    profiles::dns::record { "${fqdn}_${intf}_PTR":
-      value  => "${::facts['networking']['fqdn']}.",
+    profiles::dns::record { "${title}_PTR":
+      value  => "${hostname}.${domain}.",
       type   => 'PTR',
       record => $last_octet,
-      zone   => $::facts['arpa'][$intf]['zone'],
+      zone   => $facts['arpa'][$interface]['zone'],
       order  => $order,
     }
   }

From a7b793238ac17a37423030dfcfb09fd479e4bcdd Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 11 May 2025 16:53:34 +1000
Subject: [PATCH 70/89] fix: exclude docker0 interfaces (#282)

- docker0 is the same on many hosts

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/282
---
 site/profiles/manifests/dns/base.pp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/site/profiles/manifests/dns/base.pp b/site/profiles/manifests/dns/base.pp
index 8d10a85..a25ba08 100644
--- a/site/profiles/manifests/dns/base.pp
+++ b/site/profiles/manifests/dns/base.pp
@@ -46,8 +46,8 @@ class profiles::dns::base (
   # export dns records for client
   $facts['networking']['interfaces'].each | $interface, $data | {
 
-    # exclude those without ipv4 address, lo and anycast addresses
-    if $data['ip'] and $interface != 'lo' and $interface !~ /^anycast[0-9]$/ {
+    # exclude those without ipv4 address, lo, docker0 and anycast addresses
+    if $data['ip'] and $interface != 'lo' and $interface != 'docker0' and $interface !~ /^anycast[0-9]$/ {
 
       # use defaults for the primary_interface
       if $interface == $primary_interface {

From 90504e5b023340b9c2bb154e90a41486def7803f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 14 May 2025 20:19:18 +1000
Subject: [PATCH 71/89] chore: use alias for nameservers (#283)

- use an alias for nameservers for dhcp ranges
- move aliased nameservers to region-wide hiera

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/283
---
 hieradata/country/au/region/drw1.yaml         |  7 +++++++
 .../au/region/drw1/infra/dns/resolver.yaml    |  7 -------
 hieradata/country/au/region/syd1.yaml         |  4 ++++
 .../au/region/syd1/infra/dns/resolver.yaml    |  4 ----
 hieradata/roles/infra/dhcp/server.yaml        | 20 +++++--------------
 5 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/hieradata/country/au/region/drw1.yaml b/hieradata/country/au/region/drw1.yaml
index 703d863..712da42 100644
--- a/hieradata/country/au/region/drw1.yaml
+++ b/hieradata/country/au/region/drw1.yaml
@@ -1,2 +1,9 @@
 ---
 timezone::timezone: 'Australia/Darwin'
+profiles_dns_upstream_forwarder_unkin:
+  - 198.18.17.23
+  - 198.18.17.24
+profiles_dns_upstream_forwarder_consul:
+  - 198.18.17.34
+  - 198.18.17.35
+  - 198.18.17.36
diff --git a/hieradata/country/au/region/drw1/infra/dns/resolver.yaml b/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
index ae1582f..ed97d53 100644
--- a/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/drw1/infra/dns/resolver.yaml
@@ -1,8 +1 @@
 ---
-profiles_dns_upstream_forwarder_unkin:
-  - 198.18.17.23
-  - 198.18.17.24
-profiles_dns_upstream_forwarder_consul:
-  - 198.18.17.34
-  - 198.18.17.35
-  - 198.18.17.36
diff --git a/hieradata/country/au/region/syd1.yaml b/hieradata/country/au/region/syd1.yaml
index 4175d66..2d28c82 100644
--- a/hieradata/country/au/region/syd1.yaml
+++ b/hieradata/country/au/region/syd1.yaml
@@ -1,3 +1,7 @@
 ---
 timezone::timezone: 'Australia/Sydney'
 certbot::client::webserver: ausyd1nxvm1021.main.unkin.net
+profiles_dns_upstream_forwarder_unkin:
+  - 198.18.19.15
+profiles_dns_upstream_forwarder_consul:
+  - 198.18.19.14
diff --git a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
index 740336c..ed97d53 100644
--- a/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
+++ b/hieradata/country/au/region/syd1/infra/dns/resolver.yaml
@@ -1,5 +1 @@
 ---
-profiles_dns_upstream_forwarder_unkin:
-  - 198.18.19.15
-profiles_dns_upstream_forwarder_consul:
-  - 198.18.19.14
diff --git a/hieradata/roles/infra/dhcp/server.yaml b/hieradata/roles/infra/dhcp/server.yaml
index a186d6c..8dc6d38 100644
--- a/hieradata/roles/infra/dhcp/server.yaml
+++ b/hieradata/roles/infra/dhcp/server.yaml
@@ -15,9 +15,7 @@ profiles::dhcp::server::pools:
     range:
       - '198.18.15.200 198.18.15.220'
     gateway: 198.18.15.254
-    nameservers:
-      - 198.18.13.12
-      - 198.18.13.13
+    nameservers: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
     domain_name: main.unkin.net
     pxeserver: 198.18.13.27
   syd1-test:
@@ -26,9 +24,7 @@ profiles::dhcp::server::pools:
     range:
       - '198.18.16.200 198.18.16.220'
     gateway: 198.18.16.254
-    nameservers:
-      - 198.18.13.12
-      - 198.18.13.13
+    nameservers: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
     domain_name: main.unkin.net
     pxeserver: 198.18.13.27
   syd1-prod1:
@@ -37,9 +33,7 @@ profiles::dhcp::server::pools:
     range:
       - '198.18.13.200 198.18.13.220'
     gateway: 198.18.13.254
-    nameservers:
-      - 198.18.13.12
-      - 198.18.13.13
+    nameservers: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
     domain_name: main.unkin.net
     pxeserver: 198.18.13.27
   syd1-prod2:
@@ -48,9 +42,7 @@ profiles::dhcp::server::pools:
     range:
       - '198.18.14.200 198.18.14.220'
     gateway: 198.18.14.254
-    nameservers:
-      - 198.18.13.12
-      - 198.18.13.13
+    nameservers: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
     domain_name: main.unkin.net
     pxeserver: 198.18.13.27
   drw1-prod:
@@ -59,9 +51,7 @@ profiles::dhcp::server::pools:
     range:
       - '198.18.17.200 198.18.17.220'
     gateway: 198.18.17.1
-    nameservers:
-      - 198.18.17.7
-      - 198.18.17.8
+    nameservers: "%{alias('profiles_dns_upstream_forwarder_unkin')}"
     domain_name: main.unkin.net
     pxeserver: 198.18.13.27
 

From 2f088c461fb60b928329d8e8cb66201b3b6ea7b5 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 15 May 2025 19:29:53 +1000
Subject: [PATCH 72/89] feat: add ceph roles (#284)

- add hieradata to manage ceph repo

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/284
---
 hieradata/roles/ceph.yaml        | 17 +++++++++++++++++
 site/roles/manifests/ceph/mds.pp |  1 -
 site/roles/manifests/ceph/mon.pp |  1 -
 site/roles/manifests/ceph/osd.pp |  6 ------
 site/roles/manifests/ceph/rgw.pp |  5 +++++
 5 files changed, 22 insertions(+), 8 deletions(-)
 create mode 100644 hieradata/roles/ceph.yaml
 delete mode 100644 site/roles/manifests/ceph/osd.pp
 create mode 100644 site/roles/manifests/ceph/rgw.pp

diff --git a/hieradata/roles/ceph.yaml b/hieradata/roles/ceph.yaml
new file mode 100644
index 0000000..80faebf
--- /dev/null
+++ b/hieradata/roles/ceph.yaml
@@ -0,0 +1,17 @@
+---
+# additional repos
+profiles::yum::global::repos:
+  - ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  - ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
diff --git a/site/roles/manifests/ceph/mds.pp b/site/roles/manifests/ceph/mds.pp
index a7a6a2e..f8e0430 100644
--- a/site/roles/manifests/ceph/mds.pp
+++ b/site/roles/manifests/ceph/mds.pp
@@ -1,5 +1,4 @@
 # a role to deploy the ceph mds
-# work in progress
 class roles::ceph::mds {
     include profiles::defaults
     include profiles::base
diff --git a/site/roles/manifests/ceph/mon.pp b/site/roles/manifests/ceph/mon.pp
index b1fe65a..a1e3f2a 100644
--- a/site/roles/manifests/ceph/mon.pp
+++ b/site/roles/manifests/ceph/mon.pp
@@ -1,5 +1,4 @@
 # a role to deploy the ceph mon
-# work in progress
 class roles::ceph::mon {
     include profiles::defaults
     include profiles::base
diff --git a/site/roles/manifests/ceph/osd.pp b/site/roles/manifests/ceph/osd.pp
deleted file mode 100644
index 047718a..0000000
--- a/site/roles/manifests/ceph/osd.pp
+++ /dev/null
@@ -1,6 +0,0 @@
-# a role to deploy the ceph osd
-# work in progress
-class roles::ceph::osd {
-    include profiles::defaults
-    include profiles::base
-}
diff --git a/site/roles/manifests/ceph/rgw.pp b/site/roles/manifests/ceph/rgw.pp
new file mode 100644
index 0000000..c758ea9
--- /dev/null
+++ b/site/roles/manifests/ceph/rgw.pp
@@ -0,0 +1,5 @@
+# a role to deploy the ceph rgw
+class roles::ceph::rgw {
+    include profiles::defaults
+    include profiles::base
+}

From 1aabe2117324e4bd88516a1a9adeaaa446222e42 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 15 May 2025 19:46:59 +1000
Subject: [PATCH 73/89] feat: manage mon loopback0 (#285)

- add frrouting
- set all ceph nodes to use ospf + loopback0 + networkd
- fix ceph repos for mons

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/285
---
 .../nodes/ausyd1nxvm2040.main.unkin.net.yaml  |  2 +
 .../nodes/ausyd1nxvm2041.main.unkin.net.yaml  |  2 +
 .../nodes/ausyd1nxvm2042.main.unkin.net.yaml  |  2 +
 .../nodes/ausyd1nxvm2043.main.unkin.net.yaml  |  2 +
 .../nodes/ausyd1nxvm2044.main.unkin.net.yaml  |  2 +
 hieradata/roles/ceph.yaml                     | 47 ++++++++++++++++++-
 hieradata/roles/ceph/mon.yaml                 |  0
 7 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 hieradata/nodes/ausyd1nxvm2040.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2041.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2042.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2043.main.unkin.net.yaml
 create mode 100644 hieradata/nodes/ausyd1nxvm2044.main.unkin.net.yaml
 create mode 100644 hieradata/roles/ceph/mon.yaml

diff --git a/hieradata/nodes/ausyd1nxvm2040.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2040.main.unkin.net.yaml
new file mode 100644
index 0000000..3060aaf
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2040.main.unkin.net.yaml
@@ -0,0 +1,2 @@
+---
+networking_loopback0_ip: 198.18.23.40  # ceph-public loopback
diff --git a/hieradata/nodes/ausyd1nxvm2041.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2041.main.unkin.net.yaml
new file mode 100644
index 0000000..a97bdfd
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2041.main.unkin.net.yaml
@@ -0,0 +1,2 @@
+---
+networking_loopback0_ip: 198.18.23.41  # ceph-public loopback
diff --git a/hieradata/nodes/ausyd1nxvm2042.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2042.main.unkin.net.yaml
new file mode 100644
index 0000000..2e9c4ff
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2042.main.unkin.net.yaml
@@ -0,0 +1,2 @@
+---
+networking_loopback0_ip: 198.18.23.42   # ceph-public loopback
diff --git a/hieradata/nodes/ausyd1nxvm2043.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2043.main.unkin.net.yaml
new file mode 100644
index 0000000..e058d7e
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2043.main.unkin.net.yaml
@@ -0,0 +1,2 @@
+---
+networking_loopback0_ip: 198.18.23.43  # ceph-public loopback
diff --git a/hieradata/nodes/ausyd1nxvm2044.main.unkin.net.yaml b/hieradata/nodes/ausyd1nxvm2044.main.unkin.net.yaml
new file mode 100644
index 0000000..5d95d34
--- /dev/null
+++ b/hieradata/nodes/ausyd1nxvm2044.main.unkin.net.yaml
@@ -0,0 +1,2 @@
+---
+networking_loopback0_ip: 198.18.23.44  # ceph-public loopback
diff --git a/hieradata/roles/ceph.yaml b/hieradata/roles/ceph.yaml
index 80faebf..cf89fc5 100644
--- a/hieradata/roles/ceph.yaml
+++ b/hieradata/roles/ceph.yaml
@@ -1,17 +1,60 @@
 ---
+hiera_include:
+  - frrouting
+
+# networking
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}" # ceph public network
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
 # additional repos
 profiles::yum::global::repos:
-  - ceph:
+  ceph:
     name: ceph
     descr: ceph repository
     target: /etc/yum.repos.d/ceph.repo
     baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
     gpgkey: https://download.ceph.com/keys/release.asc
     mirrorlist: absent
-  - ceph-noarch:
+  ceph-noarch:
     name: ceph-noarch
     descr: ceph-noarch repository
     target: /etc/yum.repos.d/ceph-noarch.repo
     baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
     gpgkey: https://download.ceph.com/keys/release.asc
     mirrorlist: absent
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
diff --git a/hieradata/roles/ceph/mon.yaml b/hieradata/roles/ceph/mon.yaml
new file mode 100644
index 0000000..e69de29

From c1637d9f43f48ffd15f1ad9b51a3ec78c34a4c04 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 16 May 2025 05:56:28 +1000
Subject: [PATCH 74/89] feat: add cephadm to incus hosts (#286)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/286
---
 hieradata/roles/infra/incus/node.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 1fbe7ba..5bd1606 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -7,6 +7,7 @@ hiera_include:
 
 profiles::packages::include:
   bridge-utils: {}
+  cephadm: {}
 
 profiles::pki::vault::alt_names:
   - incus.service.consul
@@ -51,6 +52,20 @@ profiles::consul::client::node_rules:
 
 # additional repos
 profiles::yum::global::repos:
+  ceph:
+    name: ceph
+    descr: ceph repository
+    target: /etc/yum.repos.d/ceph.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
+  ceph-noarch:
+    name: ceph-noarch
+    descr: ceph-noarch repository
+    target: /etc/yum.repos.d/ceph-noarch.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/noarch
+    gpgkey: https://download.ceph.com/keys/release.asc
+    mirrorlist: absent
   frr-extras:
     name: frr-extras
     descr: frr-extras repository

From 92f0ae64b949411b6a8955d7682fbd8d0f6d02fa Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Fri, 16 May 2025 07:05:31 +1000
Subject: [PATCH 75/89] feat: enable ssh on all loopbacks (#287)

- required for cephadm to manage roles

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/287
---
 hieradata/roles/infra/incus/node.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 5bd1606..d0bfeeb 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -148,6 +148,8 @@ frrouting::daemons:
 ssh::server::options:
   ListenAddress:
     - "%{hiera('networking_loopback0_ip')}"
+    - "%{hiera('networking_loopback1_ip')}"
+    - "%{hiera('networking_loopback2_ip')}"
 
 # zfs settings
 zfs::manage_repo: false

From d9e8637ad6b08b0eb3f63333858e66ea2fdcd228 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 17 May 2025 11:14:45 +1000
Subject: [PATCH 76/89] feat: manage more ceph requirements (#288)

- add ceph-common to provide utilities for managing ceph
- add root and sysadmin ssh keys for ceph deployments

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/288
---
 hieradata/common.yaml                    |  6 ++++++
 hieradata/roles/infra/incus/node.yaml    |  6 ++++++
 site/profiles/manifests/accounts/root.pp | 18 ++++++++++++++++++
 site/profiles/manifests/base.pp          |  2 +-
 site/profiles/manifests/base/root.pp     | 13 -------------
 5 files changed, 31 insertions(+), 14 deletions(-)
 create mode 100644 site/profiles/manifests/accounts/root.pp
 delete mode 100644 site/profiles/manifests/base/root.pp

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 3bee9e1..9e0f02e 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -36,6 +36,12 @@ lookup_options:
   profiles::haproxy::server::listeners:
     merge:
       strategy: deep
+  profiles::accounts::root::sshkeys:
+    merge:
+      strategy: deep
+  profiles::accounts::sysadmin::sshkeys:
+    merge:
+      strategy: deep
   haproxy::backend:
     merge:
       strategy: deep
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index d0bfeeb..4887b89 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -8,6 +8,7 @@ hiera_include:
 profiles::packages::include:
   bridge-utils: {}
   cephadm: {}
+  ceph-common: {}
 
 profiles::pki::vault::alt_names:
   - incus.service.consul
@@ -27,6 +28,11 @@ profiles::ssh::sign::principals:
   - "%{hiera('networking_loopback1_ip')}"
   - "%{hiera('networking_loopback2_ip')}"
 
+profiles::accounts::root::sshkeys:
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEEiTQnbnfgIb2FAvrUzKkznB/Jyq06YXhP3E+Y8SmwFSeLZZPdZhKEiWRv0aY3zBIUgGsKmBXtPd8HTvQn959E6fgs3jNBtBIo76sTaR6LpNhb07tUuQDvycFlv3WZRgRu1s3RifNn0Ozfd7JPJtqjo/FGz8URtypkvOto4NnzkgOSjm1qOS6OjetBL2u+tB/h9vRDWIdKyEWqHp81aNqT9wv9MHMGBUCVNC7/WTblCsmL2rPY289dU9E/Ja5bAbNN+Lp23e8lQ+RoSeWmVIM7VCans78hLPzb2RqwNgWMBR2eStmGtHbOF1QYo3luC2GfGR7ImMfxgrR9NTu56nSHIOO+GCpWZEneIPGyLrL5vWWwhODIAJNjG6qGFeLL4PcQBYabI3fmoyrUOaMohiovLYGYs+9NK8wPOpVIP6i6CBq6RzVCjmgGq8x12dK8JhAkcoTfEcPdQwSJU/LRBFfLtRgtu1nb9BdSmotb3ESTSrXt+RYiPgAxatSSrN00qs= ceph-9a4b6eac-31d1-11f0-a634-00e04c680f5d
+profiles::accounts::sysadmin::sshkeys:
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEEiTQnbnfgIb2FAvrUzKkznB/Jyq06YXhP3E+Y8SmwFSeLZZPdZhKEiWRv0aY3zBIUgGsKmBXtPd8HTvQn959E6fgs3jNBtBIo76sTaR6LpNhb07tUuQDvycFlv3WZRgRu1s3RifNn0Ozfd7JPJtqjo/FGz8URtypkvOto4NnzkgOSjm1qOS6OjetBL2u+tB/h9vRDWIdKyEWqHp81aNqT9wv9MHMGBUCVNC7/WTblCsmL2rPY289dU9E/Ja5bAbNN+Lp23e8lQ+RoSeWmVIM7VCans78hLPzb2RqwNgWMBR2eStmGtHbOF1QYo3luC2GfGR7ImMfxgrR9NTu56nSHIOO+GCpWZEneIPGyLrL5vWWwhODIAJNjG6qGFeLL4PcQBYabI3fmoyrUOaMohiovLYGYs+9NK8wPOpVIP6i6CBq6RzVCjmgGq8x12dK8JhAkcoTfEcPdQwSJU/LRBFfLtRgtu1nb9BdSmotb3ESTSrXt+RYiPgAxatSSrN00qs= ceph-9a4b6eac-31d1-11f0-a634-00e04c680f5d
+
 # configure consul service
 consul::services:
   incus:
diff --git a/site/profiles/manifests/accounts/root.pp b/site/profiles/manifests/accounts/root.pp
new file mode 100644
index 0000000..ebae0d4
--- /dev/null
+++ b/site/profiles/manifests/accounts/root.pp
@@ -0,0 +1,18 @@
+# manage the root user
+class profiles::accounts::root (
+  Optional[Array[String]] $sshkeys = undef,
+) {
+
+  if $sshkeys {
+    accounts::user { 'root':
+      sshkeys => $sshkeys,
+    }
+  }
+
+  file {'/root/.config':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0600',
+  }
+}
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index 46df942..890fa6f 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -26,7 +26,7 @@ class profiles::base (
     include profiles::base::scripts
     include profiles::base::hosts
     include profiles::base::groups
-    include profiles::base::root
+    include profiles::accounts::root
     include profiles::accounts::sysadmin
     if $facts['virtual'] != 'lxc' {
       include profiles::ntp::client
diff --git a/site/profiles/manifests/base/root.pp b/site/profiles/manifests/base/root.pp
deleted file mode 100644
index d53951e..0000000
--- a/site/profiles/manifests/base/root.pp
+++ /dev/null
@@ -1,13 +0,0 @@
-# manage the root user
-class profiles::base::root {
-
-  # TODO
-  # for now, add some root directories
-
-  file {'/root/.config':
-    ensure => directory,
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0600',
-  }
-}

From e7d4c75192db8efb516e6a3b3db4347abca7323e Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 17 May 2025 13:50:35 +1000
Subject: [PATCH 77/89] feat: enable ssh access to enp3s0 (#289)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/289
---
 hieradata/roles/infra/incus/node.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 4887b89..dfa0db5 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -27,6 +27,7 @@ profiles::ssh::sign::principals:
   - "%{hiera('networking_loopback0_ip')}"
   - "%{hiera('networking_loopback1_ip')}"
   - "%{hiera('networking_loopback2_ip')}"
+  - "%{facts.networking.interfaces.enp3s0.ip}"
 
 profiles::accounts::root::sshkeys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEEiTQnbnfgIb2FAvrUzKkznB/Jyq06YXhP3E+Y8SmwFSeLZZPdZhKEiWRv0aY3zBIUgGsKmBXtPd8HTvQn959E6fgs3jNBtBIo76sTaR6LpNhb07tUuQDvycFlv3WZRgRu1s3RifNn0Ozfd7JPJtqjo/FGz8URtypkvOto4NnzkgOSjm1qOS6OjetBL2u+tB/h9vRDWIdKyEWqHp81aNqT9wv9MHMGBUCVNC7/WTblCsmL2rPY289dU9E/Ja5bAbNN+Lp23e8lQ+RoSeWmVIM7VCans78hLPzb2RqwNgWMBR2eStmGtHbOF1QYo3luC2GfGR7ImMfxgrR9NTu56nSHIOO+GCpWZEneIPGyLrL5vWWwhODIAJNjG6qGFeLL4PcQBYabI3fmoyrUOaMohiovLYGYs+9NK8wPOpVIP6i6CBq6RzVCjmgGq8x12dK8JhAkcoTfEcPdQwSJU/LRBFfLtRgtu1nb9BdSmotb3ESTSrXt+RYiPgAxatSSrN00qs= ceph-9a4b6eac-31d1-11f0-a634-00e04c680f5d
@@ -156,6 +157,7 @@ ssh::server::options:
     - "%{hiera('networking_loopback0_ip')}"
     - "%{hiera('networking_loopback1_ip')}"
     - "%{hiera('networking_loopback2_ip')}"
+    - "%{facts.networking.interfaces.enp3s0.ip}"
 
 # zfs settings
 zfs::manage_repo: false

From 6dcc7343e09ed6fdd203428da59652bddff60294 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 17 May 2025 14:05:25 +1000
Subject: [PATCH 78/89] feat: updated ceph ssh authorized_key (#290)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/290
---
 hieradata/roles/infra/incus/node.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index dfa0db5..72d7155 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -30,9 +30,9 @@ profiles::ssh::sign::principals:
   - "%{facts.networking.interfaces.enp3s0.ip}"
 
 profiles::accounts::root::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEEiTQnbnfgIb2FAvrUzKkznB/Jyq06YXhP3E+Y8SmwFSeLZZPdZhKEiWRv0aY3zBIUgGsKmBXtPd8HTvQn959E6fgs3jNBtBIo76sTaR6LpNhb07tUuQDvycFlv3WZRgRu1s3RifNn0Ozfd7JPJtqjo/FGz8URtypkvOto4NnzkgOSjm1qOS6OjetBL2u+tB/h9vRDWIdKyEWqHp81aNqT9wv9MHMGBUCVNC7/WTblCsmL2rPY289dU9E/Ja5bAbNN+Lp23e8lQ+RoSeWmVIM7VCans78hLPzb2RqwNgWMBR2eStmGtHbOF1QYo3luC2GfGR7ImMfxgrR9NTu56nSHIOO+GCpWZEneIPGyLrL5vWWwhODIAJNjG6qGFeLL4PcQBYabI3fmoyrUOaMohiovLYGYs+9NK8wPOpVIP6i6CBq6RzVCjmgGq8x12dK8JhAkcoTfEcPdQwSJU/LRBFfLtRgtu1nb9BdSmotb3ESTSrXt+RYiPgAxatSSrN00qs= ceph-9a4b6eac-31d1-11f0-a634-00e04c680f5d
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
 profiles::accounts::sysadmin::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEEiTQnbnfgIb2FAvrUzKkznB/Jyq06YXhP3E+Y8SmwFSeLZZPdZhKEiWRv0aY3zBIUgGsKmBXtPd8HTvQn959E6fgs3jNBtBIo76sTaR6LpNhb07tUuQDvycFlv3WZRgRu1s3RifNn0Ozfd7JPJtqjo/FGz8URtypkvOto4NnzkgOSjm1qOS6OjetBL2u+tB/h9vRDWIdKyEWqHp81aNqT9wv9MHMGBUCVNC7/WTblCsmL2rPY289dU9E/Ja5bAbNN+Lp23e8lQ+RoSeWmVIM7VCans78hLPzb2RqwNgWMBR2eStmGtHbOF1QYo3luC2GfGR7ImMfxgrR9NTu56nSHIOO+GCpWZEneIPGyLrL5vWWwhODIAJNjG6qGFeLL4PcQBYabI3fmoyrUOaMohiovLYGYs+9NK8wPOpVIP6i6CBq6RzVCjmgGq8x12dK8JhAkcoTfEcPdQwSJU/LRBFfLtRgtu1nb9BdSmotb3ESTSrXt+RYiPgAxatSSrN00qs= ceph-9a4b6eac-31d1-11f0-a634-00e04c680f5d
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
 
 # configure consul service
 consul::services:

From 89a0f329d83a9aa0832fde905a9a59c7c4b20e31 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 21 May 2025 19:58:12 +1000
Subject: [PATCH 79/89] feat: update vault url (#291)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/291
---
 hieradata/roles/infra/automation/rundeck.yaml | 2 +-
 hieradata/roles/infra/puppet/master.yaml      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hieradata/roles/infra/automation/rundeck.yaml b/hieradata/roles/infra/automation/rundeck.yaml
index f46abc1..8fc070a 100644
--- a/hieradata/roles/infra/automation/rundeck.yaml
+++ b/hieradata/roles/infra/automation/rundeck.yaml
@@ -91,7 +91,7 @@ profiles::rundeck::server::key_storage_config:
     path: 'vault'
     config:
       prefix: 'rundeck'
-      address: https://vault.query.consul:8200
+      address: https://vault.service.consul:8200
       storageBehaviour: 'vault'
       secretBackend: rundeck
       engineVersion: '2'
diff --git a/hieradata/roles/infra/puppet/master.yaml b/hieradata/roles/infra/puppet/master.yaml
index 37ebf4f..ea51ac8 100644
--- a/hieradata/roles/infra/puppet/master.yaml
+++ b/hieradata/roles/infra/puppet/master.yaml
@@ -37,7 +37,7 @@ profiles::puppet::gems::puppet:
   - 'hiera-eyaml'
 
 profiles::helpers::certmanager::vault_config:
-  addr: 'https://vault.query.consul:8200'
+  addr: 'https://vault.service.consul:8200'
   mount_point: 'pki_int'
   approle_path: 'approle'
   role_name: 'servers_default'

From 77d07672f86261948f0fd1d9c2afbc301e23e0ff Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 22 May 2025 21:06:15 +1000
Subject: [PATCH 80/89] chore: dont mount cephfs inside lxc (#292)

- lxc instances will have cephfs passed from the host
- skip cephfs mounting for lxc instances

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/292
---
 site/profiles/manifests/nomad/node.pp | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/site/profiles/manifests/nomad/node.pp b/site/profiles/manifests/nomad/node.pp
index 942b596..dfc33ff 100644
--- a/site/profiles/manifests/nomad/node.pp
+++ b/site/profiles/manifests/nomad/node.pp
@@ -33,16 +33,19 @@ class profiles::nomad::node (
 
   if $client {
 
-    include profiles::ceph::client
+    if $facts['virtual'] != 'lxc' {
+      include profiles::ceph::client
 
-    # manage the sharedvol
-    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_nomad":
-      mount       => $nomad_root,
-      keyring     => '/etc/ceph/ceph.client.nomad.keyring',
-      cephfs_name => 'nomad',
-      cephfs_fs   => 'nomadfs',
-      require     => Profiles::Ceph::Keyring['nomad'],
+      # manage the sharedvol
+      profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_nomad":
+        mount       => $nomad_root,
+        keyring     => '/etc/ceph/ceph.client.nomad.keyring',
+        cephfs_name => 'nomad',
+        cephfs_fs   => 'nomadfs',
+        require     => Profiles::Ceph::Keyring['nomad'],
+      }
     }
+
   }
 
   file { $data_dir:

From 520e8a34e09e5cc3e75b9aee36566e5d1185711a Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 24 May 2025 15:35:20 +1000
Subject: [PATCH 81/89] feat: add a nomad agent v2 role (#293)

- excludes ceph (will be passed from incus)
- excludes frrouting (will use host-networking)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/293
---
 hieradata/roles/infra/nomad/agentv2.yaml    | 55 +++++++++++++++++++++
 site/roles/manifests/infra/nomad/agentv2.pp | 12 +++++
 2 files changed, 67 insertions(+)
 create mode 100644 hieradata/roles/infra/nomad/agentv2.yaml
 create mode 100644 site/roles/manifests/infra/nomad/agentv2.pp

diff --git a/hieradata/roles/infra/nomad/agentv2.yaml b/hieradata/roles/infra/nomad/agentv2.yaml
new file mode 100644
index 0000000..629a9be
--- /dev/null
+++ b/hieradata/roles/infra/nomad/agentv2.yaml
@@ -0,0 +1,55 @@
+---
+hiera_include:
+  - docker
+  - docker::networks
+  - profiles::nomad::node
+
+docker::version: latest
+docker::curl_ensure: false
+docker::root_dir: /data/docker
+docker::ip_forward: true
+#docker::ip_masq: false
+#docker::iptables: false
+
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+
+profiles::packages::include:
+  nomad: {}
+  cni-plugins: {}
+
+profiles::nomad::node::client: true
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - client.global.nomad
+  - client.au-syd1.nomad
+  - nomad-client.service.consul
+  - nomad-client.query.consul
+  - "nomad-client.service.%{facts.country}-%{facts.region}.consul"
+
+# configure consul service
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: nomad-client
+    disposition: write
+  - resource: agent_prefix
+    segment: ''
+    disposition: read
+  - resource: node_prefix
+    segment: ''
+    disposition: write
+  - resource: service_prefix
+    segment: ''
+    disposition: write
+  - resource: key_prefix
+    segment: "nomad"
+    disposition: write
+  - resource: session_prefix
+    segment: ""
+    disposition: write
diff --git a/site/roles/manifests/infra/nomad/agentv2.pp b/site/roles/manifests/infra/nomad/agentv2.pp
new file mode 100644
index 0000000..1a5a02e
--- /dev/null
+++ b/site/roles/manifests/infra/nomad/agentv2.pp
@@ -0,0 +1,12 @@
+# a role to deploy a nomad agent, second iteration
+# using host based networking
+class roles::infra::nomad::agentv2 {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}

From 93cd02deec911e5aad991c09a93a8735a0806d66 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 24 May 2025 18:59:46 +1000
Subject: [PATCH 82/89] chore: update media roles for incus (#294)

- prevent incus roles from exporting haproxy endpoints (for now)
- incus doesnt need to mount cephfs

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/294
---
 hieradata/virtual/lxc.yaml                |  9 +++++
 site/profiles/manifests/media/jellyfin.pp | 46 ++++++++++++-----------
 2 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
index f2d7929..98c12d9 100644
--- a/hieradata/virtual/lxc.yaml
+++ b/hieradata/virtual/lxc.yaml
@@ -5,3 +5,12 @@ profiles::packages::include:
 
 # disable mlock for vault nodes on lxd/incus
 vault::disable_mlock: true
+
+hiera_exclude:
+  # exclude the media profiles which are just cephfs-mount-management
+  - profiles::media::lidar
+  - profiles::media::nzbget
+  - profiles::media::prowlarr
+  - profiles::media::radarr
+  - profiles::media::readarr
+  - profiles::media::sonarr
diff --git a/site/profiles/manifests/media/jellyfin.pp b/site/profiles/manifests/media/jellyfin.pp
index 4943a1c..c4e0525 100644
--- a/site/profiles/manifests/media/jellyfin.pp
+++ b/site/profiles/manifests/media/jellyfin.pp
@@ -13,30 +13,32 @@ class profiles::media::jellyfin (
   Boolean              $migrate_data   = true,
 ) {
 
-  include profiles::ceph::client
+  if $facts['virtual'] == 'lxc' {
+    include profiles::ceph::client
 
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
-  }
+    # manage the sharedvol
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
 
-  # export haproxy balancemember
-  profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_443":
-    service => 'be_jellyfin',
-    ports   => [443],
-    options => [
-      "cookie ${facts['networking']['hostname']}",
-      'ssl',
-      'verify none',
-      'check',
-      'inter 2s',
-      'rise 3',
-      'fall 2',
-    ]
+    # export haproxy balancemember
+    profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_443":
+      service => 'be_jellyfin',
+      ports   => [443],
+      options => [
+        "cookie ${facts['networking']['hostname']}",
+        'ssl',
+        'verify none',
+        'check',
+        'inter 2s',
+        'rise 3',
+        'fall 2',
+      ]
+    }
   }
 
   mkdir::p {[$data_dir, $lib_dir, $cache_dir, $config_dir, $log_dir]:}

From f6694599ef338359ba904339e9b1261e03ba90f2 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 24 May 2025 20:18:23 +1000
Subject: [PATCH 83/89] benvin/media_apps_incus (#295)

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/295
---
 hieradata/virtual/lxc.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
index 98c12d9..d9ed4d2 100644
--- a/hieradata/virtual/lxc.yaml
+++ b/hieradata/virtual/lxc.yaml
@@ -8,7 +8,7 @@ vault::disable_mlock: true
 
 hiera_exclude:
   # exclude the media profiles which are just cephfs-mount-management
-  - profiles::media::lidar
+  - profiles::media::lidarr
   - profiles::media::nzbget
   - profiles::media::prowlarr
   - profiles::media::radarr

From 596e498a0032234fe296700b024c900f1ccbb516 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 24 May 2025 20:23:56 +1000
Subject: [PATCH 84/89] feat: change media arr apps to hiera_include (#296)

- change profiles::media::* to be hiera_included
- this is required to enable it to be hiera_excluded on virtual == lxc

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/296
---
 hieradata/roles/apps/media/lidarr.yaml      | 1 +
 hieradata/roles/apps/media/prowlarr.yaml    | 1 +
 hieradata/roles/apps/media/radarr.yaml      | 1 +
 hieradata/roles/apps/media/readarr.yaml     | 1 +
 hieradata/roles/apps/media/sonarr.yaml      | 1 +
 site/roles/manifests/apps/media/lidarr.pp   | 1 -
 site/roles/manifests/apps/media/prowlarr.pp | 1 -
 site/roles/manifests/apps/media/radarr.pp   | 1 -
 site/roles/manifests/apps/media/readarr.pp  | 1 -
 site/roles/manifests/apps/media/sonarr.pp   | 1 -
 10 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hieradata/roles/apps/media/lidarr.yaml b/hieradata/roles/apps/media/lidarr.yaml
index 5c3b754..87911c3 100644
--- a/hieradata/roles/apps/media/lidarr.yaml
+++ b/hieradata/roles/apps/media/lidarr.yaml
@@ -2,6 +2,7 @@
 hiera_include:
   - lidarr
   - profiles::nginx::ldapauth
+  - profiles::media::lidarr
 
 # manage lidarr
 lidarr::params::user: lidarr
diff --git a/hieradata/roles/apps/media/prowlarr.yaml b/hieradata/roles/apps/media/prowlarr.yaml
index 7ee7e70..38280cb 100644
--- a/hieradata/roles/apps/media/prowlarr.yaml
+++ b/hieradata/roles/apps/media/prowlarr.yaml
@@ -2,6 +2,7 @@
 hiera_include:
   - prowlarr
   - profiles::nginx::ldapauth
+  - profiles::media::prowlarr
 
 # manage prowlarr
 prowlarr::params::user: prowlarr
diff --git a/hieradata/roles/apps/media/radarr.yaml b/hieradata/roles/apps/media/radarr.yaml
index 1cd50a4..64c9076 100644
--- a/hieradata/roles/apps/media/radarr.yaml
+++ b/hieradata/roles/apps/media/radarr.yaml
@@ -2,6 +2,7 @@
 hiera_include:
   - radarr
   - profiles::nginx::ldapauth
+  - profiles::media::radarr
 
 # manage radarr
 radarr::params::user: radarr
diff --git a/hieradata/roles/apps/media/readarr.yaml b/hieradata/roles/apps/media/readarr.yaml
index ee17dce..b8cf38c 100644
--- a/hieradata/roles/apps/media/readarr.yaml
+++ b/hieradata/roles/apps/media/readarr.yaml
@@ -2,6 +2,7 @@
 hiera_include:
   - readarr
   - profiles::nginx::ldapauth
+  - profiles::media::readarr
 
 # manage readarr
 readarr::params::user: readarr
diff --git a/hieradata/roles/apps/media/sonarr.yaml b/hieradata/roles/apps/media/sonarr.yaml
index 578bbff..32969e0 100644
--- a/hieradata/roles/apps/media/sonarr.yaml
+++ b/hieradata/roles/apps/media/sonarr.yaml
@@ -2,6 +2,7 @@
 hiera_include:
   - sonarr
   - profiles::nginx::ldapauth
+  - profiles::media::sonarr
 
 # manage sonarr
 sonarr::params::user: sonarr
diff --git a/site/roles/manifests/apps/media/lidarr.pp b/site/roles/manifests/apps/media/lidarr.pp
index 5278575..2691cc2 100644
--- a/site/roles/manifests/apps/media/lidarr.pp
+++ b/site/roles/manifests/apps/media/lidarr.pp
@@ -6,6 +6,5 @@ class roles::apps::media::lidarr {
   }else{
     include profiles::defaults
     include profiles::base
-    include profiles::media::lidarr
   }
 }
diff --git a/site/roles/manifests/apps/media/prowlarr.pp b/site/roles/manifests/apps/media/prowlarr.pp
index 03e0839..4dd5854 100644
--- a/site/roles/manifests/apps/media/prowlarr.pp
+++ b/site/roles/manifests/apps/media/prowlarr.pp
@@ -6,6 +6,5 @@ class roles::apps::media::prowlarr {
   }else{
     include profiles::defaults
     include profiles::base
-    include profiles::media::prowlarr
   }
 }
diff --git a/site/roles/manifests/apps/media/radarr.pp b/site/roles/manifests/apps/media/radarr.pp
index c94ae81..93fca24 100644
--- a/site/roles/manifests/apps/media/radarr.pp
+++ b/site/roles/manifests/apps/media/radarr.pp
@@ -6,6 +6,5 @@ class roles::apps::media::radarr {
   }else{
     include profiles::defaults
     include profiles::base
-    include profiles::media::radarr
   }
 }
diff --git a/site/roles/manifests/apps/media/readarr.pp b/site/roles/manifests/apps/media/readarr.pp
index adbd553..0dfcf55 100644
--- a/site/roles/manifests/apps/media/readarr.pp
+++ b/site/roles/manifests/apps/media/readarr.pp
@@ -6,6 +6,5 @@ class roles::apps::media::readarr {
   }else{
     include profiles::defaults
     include profiles::base
-    include profiles::media::readarr
   }
 }
diff --git a/site/roles/manifests/apps/media/sonarr.pp b/site/roles/manifests/apps/media/sonarr.pp
index 07a919c..0ceab35 100644
--- a/site/roles/manifests/apps/media/sonarr.pp
+++ b/site/roles/manifests/apps/media/sonarr.pp
@@ -6,6 +6,5 @@ class roles::apps::media::sonarr {
   }else{
     include profiles::defaults
     include profiles::base
-    include profiles::media::sonarr
   }
 }

From c0aab1087ee48fea5ff6b00af5adae5d032d84f5 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 24 May 2025 21:10:56 +1000
Subject: [PATCH 85/89] fix: readd to jellyfin_haproxy (#297)

- fix operator for jellyfin/haproxy

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/297
---
 site/profiles/manifests/media/jellyfin.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/profiles/manifests/media/jellyfin.pp b/site/profiles/manifests/media/jellyfin.pp
index c4e0525..03f0096 100644
--- a/site/profiles/manifests/media/jellyfin.pp
+++ b/site/profiles/manifests/media/jellyfin.pp
@@ -13,7 +13,7 @@ class profiles::media::jellyfin (
   Boolean              $migrate_data   = true,
 ) {
 
-  if $facts['virtual'] == 'lxc' {
+  if $facts['virtual'] != 'lxc' {
     include profiles::ceph::client
 
     # manage the sharedvol

From 1d23fef82ec2e89799e4af09e5131865d5c5955f Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 25 May 2025 20:22:00 +1000
Subject: [PATCH 86/89] feat: update settings for ceph (#298)

- enable root logins via ssh with keys
- add ssh key for ceph to root user

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/298
---
 hieradata/common.yaml                         |  1 +
 hieradata/roles/infra/incus/node.eyaml        |  2 +
 hieradata/roles/infra/incus/node.yaml         | 58 ++++++++++++++-----
 site/profiles/manifests/ceph/client.pp        | 39 ++++++++-----
 site/profiles/manifests/ceph/node.pp          | 31 ++++++++++
 site/profiles/manifests/storage/cephfsvols.pp | 36 ++++++++++++
 6 files changed, 139 insertions(+), 28 deletions(-)
 create mode 100644 hieradata/roles/infra/incus/node.eyaml
 create mode 100644 site/profiles/manifests/ceph/node.pp
 create mode 100644 site/profiles/manifests/storage/cephfsvols.pp

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 9e0f02e..48590eb 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -355,6 +355,7 @@ networking::route_defaults:
   netmask: 0.0.0.0
   network: default
 
+# FIXME these are for the proxmox ceph cluster
 profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8
 profiles::ceph::client::mons:
   - 10.18.15.1
diff --git a/hieradata/roles/infra/incus/node.eyaml b/hieradata/roles/infra/incus/node.eyaml
new file mode 100644
index 0000000..c85b8f6
--- /dev/null
+++ b/hieradata/roles/infra/incus/node.eyaml
@@ -0,0 +1,2 @@
+ceph::key::media: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAX4pXMYx0Kms74s+CCT9EnuVSq3fvlcxpkezFt/VYR0cfo8K8Sm0CE0MFEQ/sdZJ+bnycjWhnkRrT0Wj2gCcqG5+KSJPVb6OOF/zkCMEjLu7xSatKakdOmnTxJEx47Lo84dGVk+YqiTT1E5XVoKlFUvZw7iIW6BgTSORo75fC2VE8MsV0GVMIusgjqMuWgyhVUGKkPGUsGAPs5Ky7uys9tlyH5s4FiZqUKvPdeKlX0HMH5XrHuKBSJg+Nju7GLNLB+/XjBsTZrY2OMC0fzczb1EQ5KL2Pl502sE8vr1VNbXqDg7vZGzWnI60Yv2t2GUxLpUjM741NP9jZ0ovGQT8I2DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDUnyCYnAG1HFiBAef1KYKsgDCtxa4fw0zb7DFzVxjpM1n5fAabbaAxIXaOxLC8u3XM3+5CYsG1dWTIDZOo+qkS9sY=]
+ceph::key::apps: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANWnFzl98B+YVXXfj5zYF5XAFbZ8iGu2VZeF53QiYxuTmRCT2/3GI4h1WB/NYcGYgmxynEsdiJus7uRO8dplsJsm+lRNffTwjnC0oDxWI82C/QuccR+YxuywqnbHLFXxZecU6/joHewt8nIu9unmF92552pqOx30ashb30L0ptR3BYBy5e9eGjjWnMrKPM4wU5NJrpW8fb9OibMWWZ1JON9tq8QITm2USCHkVOAPCfo94Dlo+2mdQWtn0GhFnRaiTNNnKT1zqSdagOUOkkY6v3yDQdmOKtR77R8bJLo/WDxUodKOt6Oi8Iuvkkvjjk6t/u05eQTPNoVQo6blV13qoDzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDWHi5X6rS0BOznTJZH1IfBgDBFQ7cIEZWzVqrEkKAb7FXdK9U+/Z2Coda3GTZ0FPJ2SzVoAV9CXNuw4808RMsS6RU=]
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 72d7155..cd07ebc 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -4,6 +4,12 @@ hiera_include:
   - frrouting
   - incus
   - zfs
+  - profiles::ceph::node
+  - profiles::ceph::client
+  - profiles::storage::cephfsvols
+
+# FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
+python::manage_dev_package: false
 
 profiles::packages::include:
   bridge-utils: {}
@@ -25,15 +31,9 @@ profiles::ssh::sign::principals:
   - incus.query.consul
   - "incus.service.%{facts.country}-%{facts.region}.consul"
   - "%{hiera('networking_loopback0_ip')}"
-  - "%{hiera('networking_loopback1_ip')}"
-  - "%{hiera('networking_loopback2_ip')}"
+  - "%{facts.networking.interfaces.enp2s0.ip}"
   - "%{facts.networking.interfaces.enp3s0.ip}"
 
-profiles::accounts::root::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-profiles::accounts::sysadmin::sshkeys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChgO55fBXEWd8E707Zos3vTVNzeDzpRePMqzitAw939hVjfzP1jdLbuDEt7raFTmyt6yPDCbVmjp1NrMJamHIZfwbhqv0D6+sKI73W50XoyZ7xdH9t/dcOsq3oGgBPrgDurxDL8A0A40nZrEbQ9VZCRTXq843qT/P6N7ZKfa8wgtLPSVxDAKHiyQJ6j00DCGx9t7eiKQO2dJU40YNZnkwpA25tLmYQ1aKm5aUuXabxm6F6NBR4hQxsPu1U4dWUKtUzEEm8pwo42hykLMcHi0FeDoICDDwX9896J8WleeJCWgUlNX5Z99m+usqFtPbJiQwJmXl+R+8gKjCj9ir8ec+FOtaM/vFwMmjiHI8Ar1T/UOiScGpbnbdS2+LuW+N2Ca5yMFNHEarZRI8LcV0XyNT7To2Ji71TYkyeFNzz/JdZ3UBCBpTQup4LPSxOsKK2xRjOKlQ+ZhwMt4c/IB7tWcIgExH4AdI3iILTs+FxJTJ221bFhDw2nECb/BR1SnJKmwE= ceph-484b46d4-32d2-11f0-b03a-00e04c680f5d
-
 # configure consul service
 consul::services:
   incus:
@@ -108,24 +108,24 @@ networking::interfaces:
     forwarding: true
   enp3s0:
     type: physical
-    mtu: 9000
+    mtu: 1500
     txqueuelen: 10000
     forwarding: true
   loopback0:
     type: dummy
     ipaddress: "%{hiera('networking_loopback0_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback1:
     type: dummy
     ipaddress: "%{hiera('networking_loopback1_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
   loopback2:
     type: dummy
     ipaddress: "%{hiera('networking_loopback2_ip')}"
     netmask: 255.255.255.255
-    mtu: 9000
+    mtu: 1500
 
 # frrouting
 frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
@@ -155,8 +155,7 @@ frrouting::daemons:
 ssh::server::options:
   ListenAddress:
     - "%{hiera('networking_loopback0_ip')}"
-    - "%{hiera('networking_loopback1_ip')}"
-    - "%{hiera('networking_loopback2_ip')}"
+    - "%{facts.networking.interfaces.enp2s0.ip}"
     - "%{facts.networking.interfaces.enp3s0.ip}"
 
 # zfs settings
@@ -193,6 +192,39 @@ incus::server_addr: "%{hiera('networking_loopback0_ip')}"
 profiles::accounts::sysadmin::extra_groups:
   - incus-admin
 
+# manage cephfs mounts
+profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_package: false
+profiles::ceph::client::manage_ceph_paths: false
+profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+profiles::ceph::client::mons:
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+profiles::ceph::client::keyrings:
+  media:
+    key: "%{hiera('ceph::key::media')}"
+  apps:
+    key: "%{hiera('ceph::key::apps')}"
+
+profiles::storage::cephfsvols::volumes:
+  cephfsvol_media:
+    mount: "/shared/media"
+    keyring: "/etc/ceph/ceph.client.media.keyring"
+    cephfs_name: "media"
+    cephfs_fs: "mediafs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[media]"
+  cephfsvol_apps:
+    mount: "/shared/apps"
+    keyring: "/etc/ceph/ceph.client.apps.keyring"
+    cephfs_name: "apps"
+    cephfs_fs: "appfs"
+    cephfs_mon: "%{alias('profiles::ceph::client::mons')}"
+    require: "Profiles::Ceph::Keyring[apps]"
+
 # sysctl recommendations
 sysctl::base::values:
   fs.aio-max-nr:
diff --git a/site/profiles/manifests/ceph/client.pp b/site/profiles/manifests/ceph/client.pp
index 1735a19..db7187c 100644
--- a/site/profiles/manifests/ceph/client.pp
+++ b/site/profiles/manifests/ceph/client.pp
@@ -3,6 +3,9 @@ class profiles::ceph::client (
   String $fsid,
   Array[Stdlib::Host] $mons,
   Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf',
+  Boolean $manage_ceph_conf = true,
+  Boolean $manage_ceph_package = true,
+  Boolean $manage_ceph_paths = true,
   String $owner = 'ceph',
   String $group = 'ceph',
   Stdlib::Filemode $mode = '0644',
@@ -13,27 +16,33 @@ class profiles::ceph::client (
   if $facts['enc_role'] != 'roles::infra::proxmox::node' {
 
     # install the ceph client package
-    package { 'ceph-common':
-      ensure => installed,
+    if $manage_ceph_package {
+      package { 'ceph-common':
+        ensure => installed,
+      }
     }
 
     # manage the ceph directory
-    file { '/etc/ceph':
-      ensure  => directory,
-      owner   => $owner,
-      group   => $group,
-      mode    => $mode,
-      require => Package['ceph-common'],
+    if $manage_ceph_paths {
+      file { '/etc/ceph':
+        ensure  => directory,
+        owner   => $owner,
+        group   => $group,
+        mode    => $mode,
+        require => Package['ceph-common'],
+      }
     }
 
     # create a basic client config
-    file { $config_file:
-      ensure  => file,
-      owner   => $owner,
-      group   => $group,
-      mode    => $mode,
-      content => template('profiles/ceph/client.conf.erb'),
-      require => Package['ceph-common'],
+    if $manage_ceph_conf {
+      file { $config_file:
+        ensure  => file,
+        owner   => $owner,
+        group   => $group,
+        mode    => $mode,
+        content => template('profiles/ceph/client.conf.erb'),
+        require => Package['ceph-common'],
+      }
     }
 
     # manage ceph keyrings
diff --git a/site/profiles/manifests/ceph/node.pp b/site/profiles/manifests/ceph/node.pp
new file mode 100644
index 0000000..df10456
--- /dev/null
+++ b/site/profiles/manifests/ceph/node.pp
@@ -0,0 +1,31 @@
+class profiles::ceph::node (
+
+){
+
+  package {'ceph':
+    ensure => 'installed',
+  }
+
+  file {'/etc/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  file {'/var/log/ceph':
+    ensure  => directory,
+    owner   => 'ceph',
+    group   => 'ceph',
+    mode    => '0755',
+    require => Package['ceph'],
+  }
+
+  # run sudo pip3 install CherryPy==18.10.0
+  # unless:
+  # [sysadmin@prodnxsr0009 ~]$ sudo pip3.9 list | grep -i cherrypy
+  # CherryPy           18.10.0
+
+
+}
diff --git a/site/profiles/manifests/storage/cephfsvols.pp b/site/profiles/manifests/storage/cephfsvols.pp
new file mode 100644
index 0000000..eb48995
--- /dev/null
+++ b/site/profiles/manifests/storage/cephfsvols.pp
@@ -0,0 +1,36 @@
+# a class to manage the cephfsvol defines
+class profiles::storage::cephfsvols (
+  Hash[String, Hash] $volumes,
+) {
+
+  $volumes.each |String $title, Hash $params| {
+
+    $ensure        = pick($params['ensure'],        'mounted')
+    $owner         = pick($params['owner'],         'root')
+    $group         = pick($params['group'],         'root')
+    $mode          = pick($params['mode'],          '0755')
+    $mount         = $params['mount']
+    $mount_options = pick($params['mount_options'], ['noatime', 'nodiratime'])
+    $cephfs_mon    = pick($params['cephfs_mon'],    'ceph-mon.service.consul')
+    $cephfs_path   = pick($params['cephfs_path'],   '/')
+    $cephfs_name   = $params['cephfs_name']
+    $cephfs_fs     = $params['cephfs_fs']
+    $keyring       = $params['keyring']
+
+    profiles::storage::cephfsvol { $title:
+      ensure        => $ensure,
+      owner         => $owner,
+      group         => $group,
+      mode          => $mode,
+      mount         => $mount,
+      mount_options => $mount_options,
+      cephfs_mon    => $cephfs_mon,
+      cephfs_path   => $cephfs_path,
+      cephfs_name   => $cephfs_name,
+      cephfs_fs     => $cephfs_fs,
+      keyring       => $keyring,
+      # Optional metaparameters like `require`
+      *             => $params.filter |$k, $v| { $k in ['require', 'before', 'notify', 'subscribe'] },
+    }
+  }
+}

From b3347f92266fa075f89e922b5ae1588731891634 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 25 May 2025 20:27:17 +1000
Subject: [PATCH 87/89] chore: migrate media applications (#299)

- migrate media applications to new cephfs pool + incus
- enable exporting haproxy
- move ceph-client-setup to only apply to non-lxc hosts
- ensure unrar is installed for nzbget
- updated jellyfin use of data_dir
- set lxc instances for jellyfin to use /shared/apps/jellyfin

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/299
---
 hieradata/roles/apps/media/jellyfin.yaml  |  8 ++++++++
 hieradata/roles/apps/media/nzbget.yaml    |  3 +++
 hieradata/virtual/lxc.yaml                | 10 ++--------
 modules/jellyfin/manifests/params.pp      |  2 +-
 site/profiles/manifests/media/jellyfin.pp |  8 ++++----
 site/profiles/manifests/media/lidarr.pp   | 19 ++++++++++---------
 site/profiles/manifests/media/nzbget.pp   | 19 ++++++++++---------
 site/profiles/manifests/media/prowlarr.pp | 19 ++++++++++---------
 site/profiles/manifests/media/radarr.pp   | 19 ++++++++++---------
 site/profiles/manifests/media/readarr.pp  | 19 ++++++++++---------
 site/profiles/manifests/media/sonarr.pp   | 19 ++++++++++---------
 11 files changed, 78 insertions(+), 67 deletions(-)

diff --git a/hieradata/roles/apps/media/jellyfin.yaml b/hieradata/roles/apps/media/jellyfin.yaml
index 7c0a226..ba85998 100644
--- a/hieradata/roles/apps/media/jellyfin.yaml
+++ b/hieradata/roles/apps/media/jellyfin.yaml
@@ -61,3 +61,11 @@ profiles::yum::global::repos:
     baseurl: https://download1.rpmfusion.org/nonfree/el/updates/%{facts.os.release.major}/%{facts.os.architecture}
     gpgkey: https://download1.rpmfusion.org/nonfree/el/RPM-GPG-KEY-rpmfusion-nonfree-el-%{facts.os.release.major}
     mirrorlist: absent
+  unkinben:
+    name: unkinben
+    descr: unkinben repository
+    target: /etc/yum.repos.d/unkin.repo
+    baseurl: https://git.query.consul/api/packages/unkinben/rpm/el8
+    gpgkey: https://git.query.consul/api/packages/unkinben/rpm/repository.key
+    gpgcheck: false
+    mirrorlist: absent
diff --git a/hieradata/roles/apps/media/nzbget.yaml b/hieradata/roles/apps/media/nzbget.yaml
index b46cca5..7a751db 100644
--- a/hieradata/roles/apps/media/nzbget.yaml
+++ b/hieradata/roles/apps/media/nzbget.yaml
@@ -5,6 +5,9 @@ hiera_include:
   - profiles::media::nzbget
   - profiles::nginx::ldapauth
 
+profiles::packages::include:
+  unrar: {}
+
 # manage nzbget
 nzbget::params::user: nzbget
 nzbget::params::group: media
diff --git a/hieradata/virtual/lxc.yaml b/hieradata/virtual/lxc.yaml
index d9ed4d2..8eb12d0 100644
--- a/hieradata/virtual/lxc.yaml
+++ b/hieradata/virtual/lxc.yaml
@@ -6,11 +6,5 @@ profiles::packages::include:
 # disable mlock for vault nodes on lxd/incus
 vault::disable_mlock: true
 
-hiera_exclude:
-  # exclude the media profiles which are just cephfs-mount-management
-  - profiles::media::lidarr
-  - profiles::media::nzbget
-  - profiles::media::prowlarr
-  - profiles::media::radarr
-  - profiles::media::readarr
-  - profiles::media::sonarr
+# manage jellyfin changes
+profiles::media::jellyfin::data_dir: /shared/apps/jellyfin
diff --git a/modules/jellyfin/manifests/params.pp b/modules/jellyfin/manifests/params.pp
index d74a3a4..889b067 100644
--- a/modules/jellyfin/manifests/params.pp
+++ b/modules/jellyfin/manifests/params.pp
@@ -1,9 +1,9 @@
 # jellyfin params
 class jellyfin::params (
   Array[String]             $packages = [
-    'jellyfin',
     'jellyfin-web',
     'jellyfin-server',
+    'jellyfin-ffmpeg-bin',
     'SDL2',
     'ffmpeg',
     'ffmpeg-devel',
diff --git a/site/profiles/manifests/media/jellyfin.pp b/site/profiles/manifests/media/jellyfin.pp
index 03f0096..c75f360 100644
--- a/site/profiles/manifests/media/jellyfin.pp
+++ b/site/profiles/manifests/media/jellyfin.pp
@@ -2,10 +2,10 @@
 class profiles::media::jellyfin (
   Stdlib::Absolutepath $media_root     = '/shared/media',
   Stdlib::Absolutepath $data_dir       = '/data/jellyfin',
-  Stdlib::Absolutepath $lib_dir        = '/data/jellyfin/var/lib',
-  Stdlib::Absolutepath $cache_dir      = '/data/jellyfin/var/cache',
-  Stdlib::Absolutepath $config_dir     = '/data/jellyfin/etc',
-  Stdlib::Absolutepath $log_dir        = '/data/jellyfin/var/log',
+  Stdlib::Absolutepath $lib_dir        = "${data_dir}/var/lib",
+  Stdlib::Absolutepath $cache_dir      = "${data_dir}/var/cache",
+  Stdlib::Absolutepath $config_dir     = "${data_dir}/etc",
+  Stdlib::Absolutepath $log_dir        = "${data_dir}/var/log",
   Stdlib::Absolutepath $ffmpeg_path    = '/usr/local/bin/ffmpeg',
   Stdlib::Absolutepath $sysconfig_file = '/etc/sysconfig/jellyfin',
   Stdlib::Absolutepath $migration_flag = '/etc/sysconfig/jellyfin_migration_done',
diff --git a/site/profiles/manifests/media/lidarr.pp b/site/profiles/manifests/media/lidarr.pp
index 6c6a0b9..c3e7a5f 100644
--- a/site/profiles/manifests/media/lidarr.pp
+++ b/site/profiles/manifests/media/lidarr.pp
@@ -3,15 +3,16 @@ class profiles::media::lidarr (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember
diff --git a/site/profiles/manifests/media/nzbget.pp b/site/profiles/manifests/media/nzbget.pp
index f80b4c8..e4df8b3 100644
--- a/site/profiles/manifests/media/nzbget.pp
+++ b/site/profiles/manifests/media/nzbget.pp
@@ -3,15 +3,16 @@ class profiles::media::nzbget (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember
diff --git a/site/profiles/manifests/media/prowlarr.pp b/site/profiles/manifests/media/prowlarr.pp
index 87d266d..de7d50a 100644
--- a/site/profiles/manifests/media/prowlarr.pp
+++ b/site/profiles/manifests/media/prowlarr.pp
@@ -3,15 +3,16 @@ class profiles::media::prowlarr (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember
diff --git a/site/profiles/manifests/media/radarr.pp b/site/profiles/manifests/media/radarr.pp
index c28560f..cae7de3 100644
--- a/site/profiles/manifests/media/radarr.pp
+++ b/site/profiles/manifests/media/radarr.pp
@@ -3,15 +3,16 @@ class profiles::media::radarr (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember
diff --git a/site/profiles/manifests/media/readarr.pp b/site/profiles/manifests/media/readarr.pp
index a788855..425a166 100644
--- a/site/profiles/manifests/media/readarr.pp
+++ b/site/profiles/manifests/media/readarr.pp
@@ -3,15 +3,16 @@ class profiles::media::readarr (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember
diff --git a/site/profiles/manifests/media/sonarr.pp b/site/profiles/manifests/media/sonarr.pp
index 2c271bc..4946871 100644
--- a/site/profiles/manifests/media/sonarr.pp
+++ b/site/profiles/manifests/media/sonarr.pp
@@ -3,15 +3,16 @@ class profiles::media::sonarr (
   Stdlib::Absolutepath $media_root = '/shared/media',
 ) {
 
-  include profiles::ceph::client
-
-  # manage the sharedvol
-  profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
-    mount       => $media_root,
-    keyring     => '/etc/ceph/ceph.client.media.keyring',
-    cephfs_name => 'media',
-    cephfs_fs   => 'mediafs',
-    require     => Profiles::Ceph::Keyring['media'],
+  if $facts['virtual'] != 'lxc' {
+    # manage the sharedvol
+    include profiles::ceph::client
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_media":
+      mount       => $media_root,
+      keyring     => '/etc/ceph/ceph.client.media.keyring',
+      cephfs_name => 'media',
+      cephfs_fs   => 'mediafs',
+      require     => Profiles::Ceph::Keyring['media'],
+    }
   }
 
   # export haproxy balancemember

From 3d5d40f38166ccb04463c0293c0329ba91b93b83 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Tue, 27 May 2025 19:55:55 +1000
Subject: [PATCH 88/89] chore: minor jellyfin updates (#300)

- add jellyfin to video group, for access to gpu
- install intel related gpu drivers
- export lxc jellyfin to haproxy

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/300
---
 hieradata/roles/apps/media/jellyfin.yaml      |  6 +++
 site/profiles/manifests/media/jellyfin.pp     | 45 ++++++++++++-------
 .../templates/jellyfin/override.conf.erb      |  2 +-
 .../profiles/templates/jellyfin/sysconfig.erb |  4 +-
 4 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/hieradata/roles/apps/media/jellyfin.yaml b/hieradata/roles/apps/media/jellyfin.yaml
index ba85998..a1e197c 100644
--- a/hieradata/roles/apps/media/jellyfin.yaml
+++ b/hieradata/roles/apps/media/jellyfin.yaml
@@ -2,6 +2,12 @@
 hiera_include:
   - jellyfin
 
+profiles::packages::include:
+  intel-media-driver: {}
+  libva-intel-driver: {}
+  libva-intel-hybrid-driver: {}
+  intel-mediasdk: {}
+
 # manage jellyfin
 jellyfin::params::service_enable: true
 
diff --git a/site/profiles/manifests/media/jellyfin.pp b/site/profiles/manifests/media/jellyfin.pp
index c75f360..be024a4 100644
--- a/site/profiles/manifests/media/jellyfin.pp
+++ b/site/profiles/manifests/media/jellyfin.pp
@@ -6,9 +6,11 @@ class profiles::media::jellyfin (
   Stdlib::Absolutepath $cache_dir      = "${data_dir}/var/cache",
   Stdlib::Absolutepath $config_dir     = "${data_dir}/etc",
   Stdlib::Absolutepath $log_dir        = "${data_dir}/var/log",
-  Stdlib::Absolutepath $ffmpeg_path    = '/usr/local/bin/ffmpeg',
+  Stdlib::Absolutepath $ffmpeg_path    = '/usr/lib/jellyfin-ffmpeg/ffmpeg',
+  Stdlib::Absolutepath $jellyfin_web   = '/usr/share/jellyfin/web',
   Stdlib::Absolutepath $sysconfig_file = '/etc/sysconfig/jellyfin',
   Stdlib::Absolutepath $migration_flag = '/etc/sysconfig/jellyfin_migration_done',
+  Stdlib::Absolutepath $transcodes_dir = '/data/jellyfin/transcodes',
   String               $service_name   = 'jellyfin',
   Boolean              $migrate_data   = true,
 ) {
@@ -25,24 +27,25 @@ class profiles::media::jellyfin (
       require     => Profiles::Ceph::Keyring['media'],
     }
 
-    # export haproxy balancemember
-    profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_443":
-      service => 'be_jellyfin',
-      ports   => [443],
-      options => [
-        "cookie ${facts['networking']['hostname']}",
-        'ssl',
-        'verify none',
-        'check',
-        'inter 2s',
-        'rise 3',
-        'fall 2',
-      ]
-    }
   }
 
-  mkdir::p {[$data_dir, $lib_dir, $cache_dir, $config_dir, $log_dir]:}
-  -> file { [$data_dir, $lib_dir, $cache_dir, $config_dir, $log_dir]:
+  # export haproxy balancemember
+  profiles::haproxy::balancemember { "${facts['networking']['fqdn']}_443":
+    service => 'be_jellyfin',
+    ports   => [443],
+    options => [
+      "cookie ${facts['networking']['hostname']}",
+      'ssl',
+      'verify none',
+      'check',
+      'inter 2s',
+      'rise 3',
+      'fall 2',
+    ]
+  }
+
+  mkdir::p {[$data_dir, $lib_dir, $cache_dir, $config_dir, $log_dir, $transcodes_dir]:}
+  -> file { [$data_dir, $lib_dir, $cache_dir, $config_dir, $log_dir, $transcodes_dir]:
     ensure => directory,
     owner  => 'jellyfin',
     group  => 'jellyfin',
@@ -115,5 +118,13 @@ class profiles::media::jellyfin (
     ],
   }
 
+  exec {'add_jellyfin_to_video_group':
+    path    => ['/usr/bin', '/bin', '/usr/sbin', '/sbin'],
+    unless  => 'getent group video | grep -q jellyfin',
+    command => 'usermod -aG video jellyfin',
+    require => Package['jellyfin-server'],
+    before  => Service['jellyfin'],
+  }
+
   systemd::daemon_reload {"${service_name}_service":}
 }
diff --git a/site/profiles/templates/jellyfin/override.conf.erb b/site/profiles/templates/jellyfin/override.conf.erb
index 53104e7..802bec3 100644
--- a/site/profiles/templates/jellyfin/override.conf.erb
+++ b/site/profiles/templates/jellyfin/override.conf.erb
@@ -4,5 +4,5 @@
 
 [Service]
 #User = jellyfin
-EnvironmentFile = <%= @environment_file %>
+EnvironmentFile = <%= @sysconfig_file %>
 WorkingDirectory = <%= @lib_dir %>
diff --git a/site/profiles/templates/jellyfin/sysconfig.erb b/site/profiles/templates/jellyfin/sysconfig.erb
index c37b26d..3a8d5cd 100644
--- a/site/profiles/templates/jellyfin/sysconfig.erb
+++ b/site/profiles/templates/jellyfin/sysconfig.erb
@@ -21,10 +21,10 @@ JELLYFIN_LOG_DIR="<%= @log_dir %>"
 JELLYFIN_CACHE_DIR="<%= @cache_dir %>"
 
 # web client path, installed by the jellyfin-web package
-JELLYFIN_WEB_OPT="--webdir=/usr/share/jellyfin-web"
+JELLYFIN_WEB_OPT="--webdir=<%= @jellyfin_web %>"
 
 # [OPTIONAL] ffmpeg binary paths, overriding the UI-configured values
-JELLYFIN_FFMPEG_OPT="--ffmpeg=<% @ffmpeg_path %>"
+JELLYFIN_FFMPEG_OPT="--ffmpeg=<%= @ffmpeg_path %>"
 
 # [OPTIONAL] run Jellyfin as a headless service
 #JELLYFIN_SERVICE_OPT="--service"

From c10a3e49fa02e0b51ac3d7275781fc14dabcd564 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Wed, 28 May 2025 19:46:45 +1000
Subject: [PATCH 89/89] chore: add new user (#301)

- just jelly access

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/301
---
 hieradata/roles/infra/auth/glauth.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/hieradata/roles/infra/auth/glauth.yaml b/hieradata/roles/infra/auth/glauth.yaml
index d3c9799..e313e40 100644
--- a/hieradata/roles/infra/auth/glauth.yaml
+++ b/hieradata/roles/infra/auth/glauth.yaml
@@ -191,6 +191,18 @@ glauth::users:
     loginshell: '/bin/bash'
     homedir: '/home/sudobo'
     passsha256: 'a326e049c2a615226877946220a978a0a8247c569be1adcd73539b09b14136d0'
+  waewak:
+    user_name: 'waewak'
+    givenname: 'Waew'
+    sn: 'Wakul'
+    mail: 'waewak@users.main.unkin.net'
+    uidnumber: 20008
+    primarygroup: 20000
+    othergroups:
+      - 20010 # jelly
+    loginshell: '/bin/bash'
+    homedir: '/home/waewak'
+    passsha256: 'd9bb99634215fe031c3bdca94149a165192fe8384ecaa238a19354c2f760a811'
 
 glauth::services:
   svc_jellyfin: