1 Commits

Author SHA1 Message Date
unkinben 5e8cff921c vault: keep systemd-tmpfiles from reaping openbao plugin sockets
ci/woodpecker/pr/ruby-validate Pipeline was successful
ci/woodpecker/pr/puppet-lint Pipeline was successful
ci/woodpecker/pr/yamllint Pipeline was successful
ci/woodpecker/pr/bolt-validate Pipeline was successful
ci/woodpecker/pr/erb-validate Pipeline was successful
ci/woodpecker/pr/epp-validate Pipeline was successful
ci/woodpecker/pr/ruby-check Pipeline was successful
ci/woodpecker/pr/puppet-validate Pipeline was successful
go-plugin gives each secrets plugin a control socket at /tmp/pluginNNN.
The daily systemd-tmpfiles-clean deletes /tmp files older than the OS
policy, which severs the socket of a long-lived plugin while its process
keeps running. Every request then fails with rpc Unavailable / dial unix
/tmp/pluginNNN: no such file, until a plugin reload respawns it. This hit
the litellm engine (terraform-vault#112) but the risk is shared by every
OpenBao plugin (gpg, rancher, gitea, ...).

Ship a tmpfiles.d exclude (x /tmp/plugin*) on all vault servers so plugin
sockets survive the cleaner. Takes effect on the next clean, no restart.

Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT
2026-08-06 22:31:58 +10:00
24 changed files with 23 additions and 403 deletions
-39
View File
@@ -409,45 +409,6 @@ profiles::ceph::client::mons:
- 10.18.15.2
- 10.18.15.3
# de96a98f cluster topology (prodnxsr0001-0019) - single source of truth for
# /etc/ceph/ceph.conf rendered by profiles::ceph::client on the k8s (osd) and
# incus (mon/mgr/mds) roles. fsid/mons are overridden per-role in the role hiera.
# public_network is the /32 of every ceph host's ceph-public loopback.
profiles::ceph::client::cluster_public_ips:
- 198.18.23.1
- 198.18.23.2
- 198.18.23.3
- 198.18.23.4
- 198.18.23.5
- 198.18.23.6
- 198.18.23.7
- 198.18.23.8
- 198.18.23.9
- 198.18.23.10
- 198.18.23.11
- 198.18.23.12
- 198.18.23.13
- 198.18.23.14
- 198.18.23.15
- 198.18.23.16
- 198.18.23.17
- 198.18.23.18
- 198.18.23.19
profiles::ceph::client::mon_initial_members:
- prodnxsr0009
- prodnxsr0010
- prodnxsr0011
- prodnxsr0012
- prodnxsr0013
# two mds daemon instances per mon/mgr/mds host (rendered only where
# render_mds_config is true, i.e. the incus node role).
profiles::ceph::client::mds_instances:
prodnxsr0009: 2
prodnxsr0010: 2
prodnxsr0011: 2
prodnxsr0012: 2
prodnxsr0013: 2
profiles::ceph::conf::config:
global:
auth_client_required: 'cephx'
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.21 # management loopback
networking_loopback1_ip: 198.18.22.14 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.14 # ceph-public loopback
networking_1000_ip: 198.18.15.14 # 1gbe network
networking_2500_ip: 198.18.21.14 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: a4:bb:6d:a4:e5:c1
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:50:03
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.22 # management loopback
networking_loopback1_ip: 198.18.22.15 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.15 # ceph-public loopback
networking_1000_ip: 198.18.15.15 # 1gbe network
networking_2500_ip: 198.18.21.15 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: a4:bb:6d:a6:30:c4
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:4f:f4
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.23 # management loopback
networking_loopback1_ip: 198.18.22.16 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.16 # ceph-public loopback
networking_1000_ip: 198.18.15.16 # 1gbe network
networking_2500_ip: 198.18.21.16 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: a4:bb:6d:9f:22:13
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:50:0c
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.24 # management loopback
networking_loopback1_ip: 198.18.22.17 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.17 # ceph-public loopback
networking_1000_ip: 198.18.15.17 # 1gbe network
networking_2500_ip: 198.18.21.17 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: 8c:04:ba:9c:b6:08
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:50:12
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.25 # management loopback
networking_loopback1_ip: 198.18.22.18 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.18 # ceph-public loopback
networking_1000_ip: 198.18.15.18 # 1gbe network
networking_2500_ip: 198.18.21.18 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: a4:bb:6d:a4:db:94
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:4f:fa
@@ -1,13 +0,0 @@
---
networking_loopback0_ip: 198.18.19.26 # management loopback
networking_loopback1_ip: 198.18.22.19 # ceph-cluster loopback
networking_loopback2_ip: 198.18.23.19 # ceph-public loopback
networking_1000_ip: 198.18.15.19 # 1gbe network
networking_2500_ip: 198.18.21.19 # 2.5gbe network
networking_1000_iface: enp1s0
networking_2500_iface: enp2s0
networking::interfaces:
"%{hiera('networking_1000_iface')}":
mac: a4:bb:6d:a4:56:11
"%{hiera('networking_2500_iface')}":
mac: c4:62:37:0d:50:00
+1 -3
View File
@@ -203,9 +203,7 @@ profiles::accounts::sysadmin::extra_groups:
- incus-admin
# manage cephfs mounts
profiles::ceph::client::manage_ceph_conf: true
# mon/mgr/mds host: render the [mds] + [mds.*] sections
profiles::ceph::client::render_mds_config: true
profiles::ceph::client::manage_ceph_conf: false
profiles::ceph::client::manage_ceph_package: false
profiles::ceph::client::manage_ceph_paths: false
profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
+1 -3
View File
@@ -11,8 +11,6 @@ hiera_include:
# manage rke2
rke2::bootstrap_node: prodnxsr0001.main.unkin.net
rke2::join_url: https://join-k8s.service.consul:9345
# pull the airgap image bundle via artifactapi (host-reachable pre-CNI), not github directly
rke2::container_archive_source: 'https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/github/rancher/rke2/releases/download'
rke2::manage_registries: true
rke2::registries:
docker.io:
@@ -92,7 +90,7 @@ profiles::packages::include:
profiles::selinux::setenforce::mode: disabled
profiles::ceph::client::manage_ceph_conf: true
profiles::ceph::client::manage_ceph_conf: false
profiles::ceph::client::manage_ceph_package: false
profiles::ceph::client::manage_ceph_paths: false
profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
-3
View File
@@ -30,9 +30,6 @@ rke2::config_hash:
- '--node-monitor-period=4s'
protect-kernel-defaults: true
disable-kube-proxy: false
# taint the control-plane/etcd nodes so only tolerating workloads schedule here
node-taint:
- "node-role.kubernetes.io/control-plane=true:NoSchedule"
# configure consul service
consul::services:
+1 -1
View File
@@ -33,7 +33,7 @@ profiles::packages::include:
profiles::selinux::setenforce::mode: disabled
profiles::ceph::client::manage_ceph_conf: true
profiles::ceph::client::manage_ceph_conf: false
profiles::ceph::client::manage_ceph_package: false
profiles::ceph::client::manage_ceph_paths: false
profiles::ceph::client::fsid: 'de96a98f-3d23-465a-a899-86d3d67edab8'
-8
View File
@@ -40,11 +40,3 @@ profiles::packages::include:
ensure: '0.1.1'
openbao-plugin-secrets-gitea:
ensure: '0.1.0'
openbao-plugin-secrets-apptoken:
ensure: '0.1.0'
openbao-plugin-secrets-netbox:
ensure: '0.1.0'
openbao-plugin-secrets-ghp:
ensure: '0.1.0'
openbao-plugin-secrets-arrstack:
ensure: '0.2.0'
-5
View File
@@ -1,8 +1,3 @@
---
# physical hosts only (facts.virtual == 'physical'); merged 'unique' with the
# common hiera_include in profiles::base.
hiera_include:
- profiles::lldpd
profiles::packages::include:
"%{hiera('lm-sensors::package')}": {}
-104
View File
@@ -1,104 +0,0 @@
# frozen_string_literal: true
require 'facter'
require 'json'
# Exposes LLDP neighbour topology (switch/port each interface is cabled to) as
# the structured `lldp` fact, keyed by local interface. This is the only source
# of physical switch/port topology in the estate and feeds NetBox. Uses
# `lldpctl -f json0`: json0 wraps every node in an array regardless of
# cardinality, so one neighbour and many neighbours parse identically (plain
# `keyvalue` folds the neighbour's sysname into the key path, and plain `json`
# collapses single-element arrays into objects). Never raises: any error or a
# down daemon yields an empty hash so a puppet run can never break.
module LldpFact
SOCKETS = ['/run/lldpd.socket', '/var/run/lldpd.socket'].freeze
module_function
# First element of a json0 node (everything is array-wrapped), or the value
# itself if it is not an array; nil when absent.
def first(node)
node.is_a?(Array) ? node[0] : node
end
# Array form of a json0 node whatever its cardinality.
def list(node)
node.is_a?(Array) ? node : [node].compact
end
# Value string of a json0 leaf like [{ 'value' => 'x' }].
def leaf(node)
entry = first(node)
entry.is_a?(Hash) ? entry['value'] : entry
end
# Chassis MAC from its id list, preferring the entry typed 'mac'.
def chassis_mac(chassis)
ids = list(chassis['id'])
mac = ids.find { |id| id.is_a?(Hash) && id['type'] == 'mac' } || ids.first
mac.is_a?(Hash) ? mac['value'] : nil
end
# Topology record for one local interface, or nil when it has no neighbour.
def neighbour(iface)
chassis = first(iface['chassis'])
port = first(iface['port'])
return nil unless chassis && port
chassis_fields(chassis).merge(port_fields(port, first(iface['vlan'])))
end
def chassis_fields(chassis)
{
'neighbor_chassis_name' => leaf(chassis['name']),
'neighbor_chassis_mac' => chassis_mac(chassis),
'neighbor_chassis_descr' => leaf(chassis['descr'])
}
end
def port_fields(port, vlan)
port_id = first(port['id'])
vlan_h = vlan.is_a?(Hash) ? vlan : {}
{
'neighbor_port_id' => port_id.is_a?(Hash) ? port_id['value'] : port_id,
'neighbor_port_descr' => leaf(port['descr']),
'vlan_id' => vlan_h['vlan-id'],
'vlan_name' => vlan_h['value']
}
end
def interfaces(output)
lldp = first(JSON.parse(output)['lldp']) || {}
list(lldp['interface'])
end
# Map of local interface => topology record, skipping interfaces with no
# neighbour.
def collect(ifaces)
ifaces.each_with_object({}) do |iface, acc|
next unless iface.is_a?(Hash)
name = iface['name']
data = neighbour(iface)
acc[name] = data if name && data
end
end
def resolve
output = Facter::Core::Execution.execute('lldpctl -f json0 2>/dev/null', on_fail: nil)
return {} if output.to_s.empty?
collect(interfaces(output))
rescue StandardError
{}
end
end
Facter.add(:lldp) do
confine kernel: 'Linux'
confine { Facter.value(:is_virtual) == false }
confine { Facter::Core::Execution.which('lldpctl') }
confine { LldpFact::SOCKETS.any? { |path| File.exist?(path) } }
setcode { LldpFact.resolve }
end
+3 -10
View File
@@ -6,17 +6,10 @@ class rke2::install (
Stdlib::HTTPUrl $container_archive_source = $rke2::container_archive_source,
){
# versionlock rke2 before install so the lock exists before any upgrade is attempted
# versionlock rke2
yum::versionlock{"rke2-${node_type}":
ensure => present,
version => "${rke2_version}~${rke2_release}",
before => Package["rke2-${node_type}"],
}
# lock rke2-common (a strict = version dep) so the rolling latest channel can't drift it ahead of the pinned server/agent
yum::versionlock{'rke2-common':
ensure => present,
version => "${rke2_version}~${rke2_release}",
}
# install rke2
@@ -34,10 +27,10 @@ class rke2::install (
before => Service["rke2-${node_type}"],
}
# preload the airgap bundle (has the default canal CNI images) so canal starts from disk, not the mirror VIP that needs flannel first
# download required archive of containers
archive { '/var/lib/rancher/rke2/agent/images/rke2-images.linux-amd64.tar.zst':
ensure => present,
source => "${container_archive_source}/v${rke2_version}%2B${rke2_release}/rke2-images.linux-amd64.tar.zst",
source => "https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/github/rancher/rke2/releases/download/v${rke2_version}%2B${rke2_release}/rke2-images.linux-amd64.tar.zst",
require => [
Package["rke2-${node_type}"],
File['/var/lib/rancher/rke2/agent/images'],
+2 -2
View File
@@ -1,8 +1,8 @@
# rke2 params
class rke2::params (
Enum['server', 'agent'] $node_type = 'agent',
String $rke2_version = '1.33.13',
String $rke2_release = 'rke2r2',
String $rke2_version = '1.33.4',
String $rke2_release = 'rke2r1',
Stdlib::Absolutepath $config_file = '/etc/rancher/rke2/config.yaml',
Hash $config_hash = {},
Stdlib::HTTPSUrl $join_url = 'https://127.0.0.1:9345',
+3 -31
View File
@@ -2,9 +2,6 @@
class profiles::ceph::client (
String $fsid,
Array[Stdlib::Host] $mons,
# cluster topology (single source of truth: hieradata/common.yaml)
Array[Stdlib::Host] $cluster_public_ips,
Array[Stdlib::Host] $mon_initial_members,
Stdlib::Absolutepath $config_file = '/etc/ceph/ceph.conf',
Boolean $manage_ceph_conf = true,
Boolean $manage_ceph_package = true,
@@ -13,24 +10,6 @@ class profiles::ceph::client (
String $group = 'ceph',
Stdlib::Filemode $mode = '0644',
Hash $keyrings = {},
# [global] tunables (defaults match the live hand-maintained ceph.conf)
String $auth_client_required = 'cephx',
String $auth_cluster_required = 'cephx',
String $auth_service_required = 'cephx',
Boolean $mon_allow_pool_delete = true,
Boolean $ms_bind_ipv4 = true,
Boolean $ms_bind_ipv6 = false,
Integer $osd_crush_chooseleaf_type = 1,
Integer $osd_pool_default_min_size = 2,
Integer $osd_pool_default_size = 3,
Integer $osd_pool_default_pg_num = 128,
# mds config sections; only rendered on mon/mgr/mds hosts (render_mds_config)
Boolean $render_mds_config = false,
Hash[String, Integer] $mds_instances = {},
Hash $mds_common = {
'keyring' => '/var/lib/ceph/mds/ceph-$id/keyring',
'mds_standby_replay' => true,
},
) {
# dont run this on proxmox nodes
@@ -43,13 +22,6 @@ class profiles::ceph::client (
}
}
# only depend on the package when this class manages it; on the ceph
# hosts the package is delivered by cephadm / profiles::packages instead.
$config_require = $manage_ceph_package ? {
true => Package['ceph-common'],
default => undef,
}
# manage the ceph directory
if $manage_ceph_paths {
file { '/etc/ceph':
@@ -57,11 +29,11 @@ class profiles::ceph::client (
owner => $owner,
group => $group,
mode => $mode,
require => $config_require,
require => Package['ceph-common'],
}
}
# render /etc/ceph/ceph.conf from cluster topology in hiera
# create a basic client config
if $manage_ceph_conf {
file { $config_file:
ensure => file,
@@ -69,7 +41,7 @@ class profiles::ceph::client (
group => $group,
mode => $mode,
content => template('profiles/ceph/client.conf.erb'),
require => $config_require,
require => Package['ceph-common'],
}
}
-3
View File
@@ -2,9 +2,6 @@ class profiles::ceph::osd (
Boolean $ensure_running = true,
) {
# tune the I/O scheduler on the disks backing ceph OSDs
include profiles::ceph::osd_scheduler
if $ensure_running and $facts['is_ceph_osd'] {
$facts['ceph_services']['osd'].each |String $svc| {
service { $svc:
@@ -1,32 +0,0 @@
class profiles::ceph::osd_scheduler (
String[1] $scheduler = 'none',
) {
$devices = $facts['ceph_osd_devices']
# no-op where the fact is absent/empty (VMs, non-OSD hosts have no ceph PVs)
if $devices =~ Array[String[1], 1] {
# strip /dev/ so the rule matches the udev KERNEL sysname (e.g. sda)
$kernel_names = $devices.map |$dev| { regsubst($dev, '^.*/', '') }
$sysname_matches = $kernel_names.map |$name| { "--sysname-match=${name}" }
file { '/etc/udev/rules.d/60-ceph-osd-scheduler.rules':
ensure => file,
owner => 'root',
group => 'root',
mode => '0644',
content => template('profiles/ceph/osd-scheduler.rules.erb'),
notify => Exec['ceph-osd-scheduler-reload'],
}
# apply immediately; udev re-applies on reboot and device re-add
$trigger = "udevadm trigger --subsystem-match=block --action=change ${join($sysname_matches, ' ')}"
exec { 'ceph-osd-scheduler-reload':
command => "udevadm control --reload-rules && ${trigger}",
path => ['/usr/bin', '/bin', '/usr/sbin', '/sbin'],
refreshonly => true,
}
}
}
-33
View File
@@ -1,33 +0,0 @@
# profiles::lldpd
#
# Runs lldpd on physical hosts so each machine learns its switch/port topology
# via LLDP. The `lldp` fact exposes that neighbour data for NetBox. Assigned
# via hiera_include from hieradata/virtual/physical.yaml (physicals only); the
# lldpd.service ships disabled, so it is explicitly enabled and started here.
class profiles::lldpd (
Boolean $enabled = true,
String $package = 'lldpd',
String $service = 'lldpd',
){
if $enabled {
package { $package:
ensure => installed,
}
service { $service:
ensure => running,
enable => true,
subscribe => Package[$package],
}
} else {
service { $service:
ensure => stopped,
enable => false,
}
package { $package:
ensure => absent,
}
}
}
+1 -2
View File
@@ -60,11 +60,10 @@ class profiles::puppet::agent (
require => Yumrepo[$use_yumrepo],
}
# versionlock puppet-agent before install so the lock exists before any upgrade is attempted
# versionlock puppet-agent
yum::versionlock{$use_package:
ensure => $agent_versionlock_ensure,
version => $agent_versionlock_version,
before => Package[$use_package],
}
}
'Debian': {
+9 -14
View File
@@ -122,6 +122,15 @@ class profiles::vault::server (
]
}
# go-plugin creates each secrets plugin's control socket as /tmp/pluginNNN;
# systemd-tmpfiles-clean reaps files aged past the /tmp policy, severing the
# socket of a long-lived plugin and orphaning the still-running process
# (every request then fails: rpc Unavailable, dial unix /tmp/pluginNNN: no
# such file). Exclude the sockets from cleaning for all plugins.
systemd::tmpfile { 'openbao-plugin-sockets.conf':
content => "x /tmp/plugin*\n",
}
# ensure the vault audit log exists
file { $audit_log:
ensure => 'file',
@@ -130,20 +139,6 @@ class profiles::vault::server (
mode => '0600',
}
# go-plugin creates each secrets plugin's control socket under TMPDIR
# (/tmp/pluginNNN by default); systemd-tmpfiles-clean reaps aged /tmp files
# and severs the socket of a long-lived plugin, orphaning the process. Point
# TMPDIR at a per-start RuntimeDirectory on /run (tmpfs, no age cleanup).
systemd::manage_dropin { 'plugin-tmpdir.conf':
unit => 'vault.service',
service_entry => {
'RuntimeDirectory' => 'vault-plugins',
'RuntimeDirectoryMode' => '0700',
'Environment' => 'TMPDIR=/run/vault-plugins',
},
notify => Service['vault'],
}
service { 'vault':
ensure => true,
enable => true,
+2 -27
View File
@@ -1,28 +1,3 @@
[global]
auth_client_required = <%= @auth_client_required %>
auth_cluster_required = <%= @auth_cluster_required %>
auth_service_required = <%= @auth_service_required %>
fsid = <%= @fsid %>
mon_allow_pool_delete = <%= @mon_allow_pool_delete %>
mon_initial_members = <%= @mon_initial_members.join(',') %>
mon_host = <%= @mons.join(',') %>
ms_bind_ipv4 = <%= @ms_bind_ipv4 %>
ms_bind_ipv6 = <%= @ms_bind_ipv6 %>
osd_crush_chooseleaf_type = <%= @osd_crush_chooseleaf_type %>
osd_pool_default_min_size = <%= @osd_pool_default_min_size %>
osd_pool_default_size = <%= @osd_pool_default_size %>
osd_pool_default_pg_num = <%= @osd_pool_default_pg_num %>
public_network = <%= @cluster_public_ips.map { |ip| "#{ip}/32" }.join(',') %>
<% if @render_mds_config -%>
[mds]
keyring = <%= @mds_common['keyring'] %>
mds_standby_replay = <%= @mds_common['mds_standby_replay'] %>
<% @mds_instances.sort.each do |host, count| -%>
<% (1..count).each do |instance| -%>
[mds.<%= host %>-<%= instance %>]
host = <%= host %>
<% end -%>
<% end -%>
<% end -%>
fsid = <%= @fsid %>
mon_host = <%= @mons.join(' ') %>
@@ -1,5 +0,0 @@
# Managed by puppet (profiles::ceph::osd_scheduler).
# Set the I/O scheduler to <%= @scheduler %> on ceph OSD block devices.
<% @kernel_names.sort.each do |dev| -%>
ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="<%= dev %>", ATTR{queue/scheduler}="<%= @scheduler %>"
<% end -%>