Bump openbao-plugin-secrets-arrstack to 0.2.0 (#522)

## Why

Plugin 0.2.0 adds per-role method scoping; the binary must land on the vault nodes before the catalog sha256 bump in terraform-vault.

## How

- Bump the `openbao-plugin-secrets-arrstack` pin in `hieradata/roles/infra/storage/vault.yaml` from `0.1.0` to `0.2.0`.

Reviewed-on: #522
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

hieradata/common.yaml

@@ -409,6 +409,45 @@ profiles::ceph::client::mons:
   - 10.18.15.2
   - 10.18.15.3
 
+# de96a98f cluster topology (prodnxsr0001-0019) - single source of truth for
+# /etc/ceph/ceph.conf rendered by profiles::ceph::client on the k8s (osd) and
+# incus (mon/mgr/mds) roles. fsid/mons are overridden per-role in the role hiera.
+# public_network is the /32 of every ceph host's ceph-public loopback.
+profiles::ceph::client::cluster_public_ips:
+  - 198.18.23.1
+  - 198.18.23.2
+  - 198.18.23.3
+  - 198.18.23.4
+  - 198.18.23.5
+  - 198.18.23.6
+  - 198.18.23.7
+  - 198.18.23.8
+  - 198.18.23.9
+  - 198.18.23.10
+  - 198.18.23.11
+  - 198.18.23.12
+  - 198.18.23.13
+  - 198.18.23.14
+  - 198.18.23.15
+  - 198.18.23.16
+  - 198.18.23.17
+  - 198.18.23.18
+  - 198.18.23.19
+profiles::ceph::client::mon_initial_members:
+  - prodnxsr0009
+  - prodnxsr0010
+  - prodnxsr0011
+  - prodnxsr0012
+  - prodnxsr0013
+# two mds daemon instances per mon/mgr/mds host (rendered only where
+# render_mds_config is true, i.e. the incus node role).
+profiles::ceph::client::mds_instances:
+  prodnxsr0009: 2
+  prodnxsr0010: 2
+  prodnxsr0011: 2
+  prodnxsr0012: 2
+  prodnxsr0013: 2
+
 profiles::ceph::conf::config:
   global:
     auth_client_required: 'cephx'

hieradata/roles/infra/incus/node.yaml

@@ -203,7 +203,9 @@ profiles::accounts::sysadmin::extra_groups:
   - incus-admin
 
 # manage cephfs mounts
-profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_conf: true
+# mon/mgr/mds host: render the [mds] + [mds.*] sections
+profiles::ceph::client::render_mds_config: true
 profiles::ceph::client::manage_ceph_package: false
 profiles::ceph::client::manage_ceph_paths: false
 profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'

hieradata/roles/infra/k8s.yaml

@@ -11,6 +11,8 @@ hiera_include:
 # manage rke2
 rke2::bootstrap_node: prodnxsr0001.main.unkin.net
 rke2::join_url: https://join-k8s.service.consul:9345
+# pull the airgap image bundle via artifactapi (host-reachable pre-CNI), not github directly
+rke2::container_archive_source: 'https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/github/rancher/rke2/releases/download'
 rke2::manage_registries: true
 rke2::registries:
   docker.io:
@@ -90,7 +92,7 @@ profiles::packages::include:
 
 profiles::selinux::setenforce::mode: disabled
 
-profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_conf: true
 profiles::ceph::client::manage_ceph_package: false
 profiles::ceph::client::manage_ceph_paths: false
 profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'

hieradata/roles/infra/k8s/control.yaml

@@ -30,6 +30,9 @@ rke2::config_hash:
     - '--node-monitor-period=4s'
   protect-kernel-defaults: true
   disable-kube-proxy: false
+  # taint the control-plane/etcd nodes so only tolerating workloads schedule here
+  node-taint:
+    - "node-role.kubernetes.io/control-plane=true:NoSchedule"
 
 # configure consul service
 consul::services:

hieradata/roles/infra/k8s/node.yaml

@@ -33,7 +33,7 @@ profiles::packages::include:
 
 profiles::selinux::setenforce::mode: disabled
 
-profiles::ceph::client::manage_ceph_conf: false
+profiles::ceph::client::manage_ceph_conf: true
 profiles::ceph::client::manage_ceph_package: false
 profiles::ceph::client::manage_ceph_paths: false
 profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'

hieradata/roles/infra/storage/vault.yaml

@@ -40,3 +40,11 @@ profiles::packages::include:
     ensure: '0.1.1'
   openbao-plugin-secrets-gitea:
     ensure: '0.1.0'
+  openbao-plugin-secrets-apptoken:
+    ensure: '0.1.0'
+  openbao-plugin-secrets-netbox:
+    ensure: '0.1.0'
+  openbao-plugin-secrets-ghp:
+    ensure: '0.1.0'
+  openbao-plugin-secrets-arrstack:
+    ensure: '0.2.0'

hieradata/virtual/physical.yaml

@@ -1,3 +1,8 @@
 ---
+# physical hosts only (facts.virtual == 'physical'); merged 'unique' with the
+# common hiera_include in profiles::base.
+hiera_include:
+  - profiles::lldpd
+
 profiles::packages::include:
   "%{hiera('lm-sensors::package')}": {}

modules/libs/lib/facter/lldp.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'facter'
+require 'json'
+
+# Exposes LLDP neighbour topology (switch/port each interface is cabled to) as
+# the structured `lldp` fact, keyed by local interface. This is the only source
+# of physical switch/port topology in the estate and feeds NetBox. Uses
+# `lldpctl -f json0`: json0 wraps every node in an array regardless of
+# cardinality, so one neighbour and many neighbours parse identically (plain
+# `keyvalue` folds the neighbour's sysname into the key path, and plain `json`
+# collapses single-element arrays into objects). Never raises: any error or a
+# down daemon yields an empty hash so a puppet run can never break.
+module LldpFact
+  SOCKETS = ['/run/lldpd.socket', '/var/run/lldpd.socket'].freeze
+
+  module_function
+
+  # First element of a json0 node (everything is array-wrapped), or the value
+  # itself if it is not an array; nil when absent.
+  def first(node)
+    node.is_a?(Array) ? node[0] : node
+  end
+
+  # Array form of a json0 node whatever its cardinality.
+  def list(node)
+    node.is_a?(Array) ? node : [node].compact
+  end
+
+  # Value string of a json0 leaf like [{ 'value' => 'x' }].
+  def leaf(node)
+    entry = first(node)
+    entry.is_a?(Hash) ? entry['value'] : entry
+  end
+
+  # Chassis MAC from its id list, preferring the entry typed 'mac'.
+  def chassis_mac(chassis)
+    ids = list(chassis['id'])
+    mac = ids.find { |id| id.is_a?(Hash) && id['type'] == 'mac' } || ids.first
+    mac.is_a?(Hash) ? mac['value'] : nil
+  end
+
+  # Topology record for one local interface, or nil when it has no neighbour.
+  def neighbour(iface)
+    chassis = first(iface['chassis'])
+    port    = first(iface['port'])
+    return nil unless chassis && port
+
+    chassis_fields(chassis).merge(port_fields(port, first(iface['vlan'])))
+  end
+
+  def chassis_fields(chassis)
+    {
+      'neighbor_chassis_name' => leaf(chassis['name']),
+      'neighbor_chassis_mac' => chassis_mac(chassis),
+      'neighbor_chassis_descr' => leaf(chassis['descr'])
+    }
+  end
+
+  def port_fields(port, vlan)
+    port_id = first(port['id'])
+    vlan_h  = vlan.is_a?(Hash) ? vlan : {}
+    {
+      'neighbor_port_id' => port_id.is_a?(Hash) ? port_id['value'] : port_id,
+      'neighbor_port_descr' => leaf(port['descr']),
+      'vlan_id' => vlan_h['vlan-id'],
+      'vlan_name' => vlan_h['value']
+    }
+  end
+
+  def interfaces(output)
+    lldp = first(JSON.parse(output)['lldp']) || {}
+    list(lldp['interface'])
+  end
+
+  # Map of local interface => topology record, skipping interfaces with no
+  # neighbour.
+  def collect(ifaces)
+    ifaces.each_with_object({}) do |iface, acc|
+      next unless iface.is_a?(Hash)
+
+      name = iface['name']
+      data = neighbour(iface)
+      acc[name] = data if name && data
+    end
+  end
+
+  def resolve
+    output = Facter::Core::Execution.execute('lldpctl -f json0 2>/dev/null', on_fail: nil)
+    return {} if output.to_s.empty?
+
+    collect(interfaces(output))
+  rescue StandardError
+    {}
+  end
+end
+
+Facter.add(:lldp) do
+  confine kernel: 'Linux'
+  confine { Facter.value(:is_virtual) == false }
+  confine { Facter::Core::Execution.which('lldpctl') }
+  confine { LldpFact::SOCKETS.any? { |path| File.exist?(path) } }
+  setcode { LldpFact.resolve }
+end

modules/rke2/manifests/install.pp

@@ -6,10 +6,17 @@ class rke2::install (
   Stdlib::HTTPUrl         $container_archive_source = $rke2::container_archive_source,
 ){
 
-  # versionlock rke2
+  # versionlock rke2 before install so the lock exists before any upgrade is attempted
   yum::versionlock{"rke2-${node_type}":
     ensure  => present,
     version => "${rke2_version}~${rke2_release}",
+    before  => Package["rke2-${node_type}"],
+  }
+
+  # lock rke2-common (a strict = version dep) so the rolling latest channel can't drift it ahead of the pinned server/agent
+  yum::versionlock{'rke2-common':
+    ensure  => present,
+    version => "${rke2_version}~${rke2_release}",
   }
 
   # install rke2
@@ -27,18 +34,15 @@ class rke2::install (
     before  => Service["rke2-${node_type}"],
   }
 
-  # preload airgap tarballs (incl. canal CNI) so flannel/calico start from disk, not the mirror VIP that needs flannel to be reachable
-  $image_archives = ['rke2-images.linux-amd64.tar.zst', 'rke2-images-canal.linux-amd64.tar.zst']
-  $image_archives.each |String $archive_file| {
-    archive { "/var/lib/rancher/rke2/agent/images/${archive_file}":
-      ensure  => present,
-      source  => "https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/github/rancher/rke2/releases/download/v${rke2_version}%2B${rke2_release}/${archive_file}",
-      require => [
-        Package["rke2-${node_type}"],
-        File['/var/lib/rancher/rke2/agent/images'],
-      ],
-      before  => Service["rke2-${node_type}"],
-    }
+  # preload the airgap bundle (has the default canal CNI images) so canal starts from disk, not the mirror VIP that needs flannel first
+  archive { '/var/lib/rancher/rke2/agent/images/rke2-images.linux-amd64.tar.zst':
+    ensure  => present,
+    source  => "${container_archive_source}/v${rke2_version}%2B${rke2_release}/rke2-images.linux-amd64.tar.zst",
+    require => [
+      Package["rke2-${node_type}"],
+      File['/var/lib/rancher/rke2/agent/images'],
+    ],
+    before  => Service["rke2-${node_type}"],
   }
 
   # ensure the images cache file exists

modules/rke2/manifests/params.pp

@@ -1,8 +1,8 @@
 # rke2 params
 class rke2::params (
   Enum['server', 'agent'] $node_type = 'agent',
-  String                  $rke2_version = '1.33.11',
-  String                  $rke2_release = 'rke2r1',
+  String                  $rke2_version = '1.33.13',
+  String                  $rke2_release = 'rke2r2',
   Stdlib::Absolutepath    $config_file = '/etc/rancher/rke2/config.yaml',
   Hash                    $config_hash = {},
   Stdlib::HTTPSUrl        $join_url = 'https://127.0.0.1:9345',

site/profiles/manifests/ceph/client.pp

@@ -2,6 +2,9 @@
 class profiles::ceph::client (
   String $fsid,
   Array[Stdlib::Host] $mons,
+  # cluster topology (single source of truth: hieradata/common.yaml)
+  Array[Stdlib::Host] $cluster_public_ips,
+  Array[Stdlib::Host] $mon_initial_members,
   Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf',
   Boolean $manage_ceph_conf = true,
   Boolean $manage_ceph_package = true,
@@ -10,6 +13,24 @@ class profiles::ceph::client (
   String $group = 'ceph',
   Stdlib::Filemode $mode = '0644',
   Hash $keyrings = {},
+  # [global] tunables (defaults match the live hand-maintained ceph.conf)
+  String $auth_client_required = 'cephx',
+  String $auth_cluster_required = 'cephx',
+  String $auth_service_required = 'cephx',
+  Boolean $mon_allow_pool_delete = true,
+  Boolean $ms_bind_ipv4 = true,
+  Boolean $ms_bind_ipv6 = false,
+  Integer $osd_crush_chooseleaf_type = 1,
+  Integer $osd_pool_default_min_size = 2,
+  Integer $osd_pool_default_size = 3,
+  Integer $osd_pool_default_pg_num = 128,
+  # mds config sections; only rendered on mon/mgr/mds hosts (render_mds_config)
+  Boolean $render_mds_config = false,
+  Hash[String, Integer] $mds_instances = {},
+  Hash $mds_common = {
+    'keyring'            => '/var/lib/ceph/mds/ceph-$id/keyring',
+    'mds_standby_replay' => true,
+  },
 ) {
 
   # dont run this on proxmox nodes
@@ -22,6 +43,13 @@ class profiles::ceph::client (
       }
     }
 
+    # only depend on the package when this class manages it; on the ceph
+    # hosts the package is delivered by cephadm / profiles::packages instead.
+    $config_require = $manage_ceph_package ? {
+      true    => Package['ceph-common'],
+      default => undef,
+    }
+
     # manage the ceph directory
     if $manage_ceph_paths {
       file { '/etc/ceph':
@@ -29,11 +57,11 @@ class profiles::ceph::client (
         owner   => $owner,
         group   => $group,
         mode    => $mode,
-        require => Package['ceph-common'],
+        require => $config_require,
       }
     }
 
-    # create a basic client config
+    # render /etc/ceph/ceph.conf from cluster topology in hiera
     if $manage_ceph_conf {
       file { $config_file:
         ensure  => file,
@@ -41,7 +69,7 @@ class profiles::ceph::client (
         group   => $group,
         mode    => $mode,
         content => template('profiles/ceph/client.conf.erb'),
-        require => Package['ceph-common'],
+        require => $config_require,
       }
     }
 

site/profiles/manifests/ceph/osd.pp

@@ -2,6 +2,9 @@ class profiles::ceph::osd (
   Boolean $ensure_running = true,
 ) {
 
+  # tune the I/O scheduler on the disks backing ceph OSDs
+  include profiles::ceph::osd_scheduler
+
   if $ensure_running and $facts['is_ceph_osd'] {
     $facts['ceph_services']['osd'].each |String $svc| {
       service { $svc:

site/profiles/manifests/ceph/osd_scheduler.pp

@@ -0,0 +1,32 @@
+class profiles::ceph::osd_scheduler (
+  String[1] $scheduler = 'none',
+) {
+
+  $devices = $facts['ceph_osd_devices']
+
+  # no-op where the fact is absent/empty (VMs, non-OSD hosts have no ceph PVs)
+  if $devices =~ Array[String[1], 1] {
+
+    # strip /dev/ so the rule matches the udev KERNEL sysname (e.g. sda)
+    $kernel_names    = $devices.map |$dev| { regsubst($dev, '^.*/', '') }
+    $sysname_matches = $kernel_names.map |$name| { "--sysname-match=${name}" }
+
+    file { '/etc/udev/rules.d/60-ceph-osd-scheduler.rules':
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('profiles/ceph/osd-scheduler.rules.erb'),
+      notify  => Exec['ceph-osd-scheduler-reload'],
+    }
+
+    # apply immediately; udev re-applies on reboot and device re-add
+    $trigger = "udevadm trigger --subsystem-match=block --action=change ${join($sysname_matches, ' ')}"
+
+    exec { 'ceph-osd-scheduler-reload':
+      command     => "udevadm control --reload-rules && ${trigger}",
+      path        => ['/usr/bin', '/bin', '/usr/sbin', '/sbin'],
+      refreshonly => true,
+    }
+  }
+}

site/profiles/manifests/lldpd.pp

@@ -0,0 +1,33 @@
+# profiles::lldpd
+#
+# Runs lldpd on physical hosts so each machine learns its switch/port topology
+# via LLDP. The `lldp` fact exposes that neighbour data for NetBox. Assigned
+# via hiera_include from hieradata/virtual/physical.yaml (physicals only); the
+# lldpd.service ships disabled, so it is explicitly enabled and started here.
+class profiles::lldpd (
+  Boolean $enabled = true,
+  String  $package = 'lldpd',
+  String  $service = 'lldpd',
+){
+
+  if $enabled {
+    package { $package:
+      ensure => installed,
+    }
+
+    service { $service:
+      ensure    => running,
+      enable    => true,
+      subscribe => Package[$package],
+    }
+  } else {
+    service { $service:
+      ensure => stopped,
+      enable => false,
+    }
+
+    package { $package:
+      ensure => absent,
+    }
+  }
+}

site/profiles/manifests/puppet/agent.pp

@@ -60,10 +60,11 @@ class profiles::puppet::agent (
         require => Yumrepo[$use_yumrepo],
       }
 
-      # versionlock puppet-agent
+      # versionlock puppet-agent before install so the lock exists before any upgrade is attempted
       yum::versionlock{$use_package:
         ensure  => $agent_versionlock_ensure,
         version => $agent_versionlock_version,
+        before  => Package[$use_package],
       }
     }
     'Debian': {

site/profiles/manifests/vault/server.pp

@@ -130,6 +130,20 @@ class profiles::vault::server (
       mode   => '0600',
     }
 
+    # go-plugin creates each secrets plugin's control socket under TMPDIR
+    # (/tmp/pluginNNN by default); systemd-tmpfiles-clean reaps aged /tmp files
+    # and severs the socket of a long-lived plugin, orphaning the process. Point
+    # TMPDIR at a per-start RuntimeDirectory on /run (tmpfs, no age cleanup).
+    systemd::manage_dropin { 'plugin-tmpdir.conf':
+      unit          => 'vault.service',
+      service_entry => {
+        'RuntimeDirectory'     => 'vault-plugins',
+        'RuntimeDirectoryMode' => '0700',
+        'Environment'          => 'TMPDIR=/run/vault-plugins',
+      },
+      notify        => Service['vault'],
+    }
+
     service { 'vault':
       ensure    => true,
       enable    => true,

site/profiles/templates/ceph/client.conf.erb

@@ -1,3 +1,28 @@
 [global]
-	fsid = <%= @fsid %>
-	mon_host = <%= @mons.join(' ') %>
+  auth_client_required = <%= @auth_client_required %>
+  auth_cluster_required = <%= @auth_cluster_required %>
+  auth_service_required = <%= @auth_service_required %>
+  fsid = <%= @fsid %>
+  mon_allow_pool_delete = <%= @mon_allow_pool_delete %>
+  mon_initial_members = <%= @mon_initial_members.join(',') %>
+  mon_host = <%= @mons.join(',') %>
+  ms_bind_ipv4 = <%= @ms_bind_ipv4 %>
+  ms_bind_ipv6 = <%= @ms_bind_ipv6 %>
+  osd_crush_chooseleaf_type = <%= @osd_crush_chooseleaf_type %>
+  osd_pool_default_min_size = <%= @osd_pool_default_min_size %>
+  osd_pool_default_size = <%= @osd_pool_default_size %>
+  osd_pool_default_pg_num = <%= @osd_pool_default_pg_num %>
+  public_network = <%= @cluster_public_ips.map { |ip| "#{ip}/32" }.join(',') %>
+<% if @render_mds_config -%>
+
+[mds]
+  keyring = <%= @mds_common['keyring'] %>
+  mds_standby_replay = <%= @mds_common['mds_standby_replay'] %>
+<% @mds_instances.sort.each do |host, count| -%>
+<% (1..count).each do |instance| -%>
+
+[mds.<%= host %>-<%= instance %>]
+  host = <%= host %>
+<% end -%>
+<% end -%>
+<% end -%>

site/profiles/templates/ceph/osd-scheduler.rules.erb

@@ -0,0 +1,5 @@
+# Managed by puppet (profiles::ceph::osd_scheduler).
+# Set the I/O scheduler to <%= @scheduler %> on ceph OSD block devices.
+<% @kernel_names.sort.each do |dev| -%>
+ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="<%= dev %>", ATTR{queue/scheduler}="<%= @scheduler %>"
+<% end -%>