Files
puppet-prod/site/profiles/manifests/vault/server.pp
T
unkinben 7102ef2b34 vault: move openbao plugin sockets off /tmp onto /run (#509)
## Why

The litellm secrets engine on the OpenBao cluster died with `rpc Unavailable / dial unix /tmp/pluginNNN: no such file` (terraform-vault#112), fixed only by a manual `sys/plugins/reload/backend`.

Root cause (post-incident log audit): go-plugin puts each plugin's control socket under the process TMPDIR — `/tmp/pluginNNN` — and `vault.service` runs without PrivateTmp. The daily `systemd-tmpfiles-clean` reaps aged `/tmp` files; bao ran a single systemd invocation for 3+ weeks, so the socket long outlived the `/tmp` cleanup age and got deleted out from under the still-running plugin process (no panic/OOM/signal/exit in the bao journal — the process was healthy, just unreachable). The risk is shared by every OpenBao plugin (gpg, rancher, gitea, ...), not just litellm.

Supersedes the earlier tmpfiles-exclude approach (#508, closed) with the permanent fix: move the sockets off `/tmp` entirely.

## Change

- Add a `vault.service` drop-in (`systemd::manage_dropin`) that sets `Environment=TMPDIR=/run/vault-plugins` and `RuntimeDirectory=vault-plugins` (mode 0700).
- Point plugin sockets at `/run` (tmpfs, no age-based cleanup); `RuntimeDirectory` creates/owns the dir per service start.
- Notify a vault service restart (module handles daemon-reload) so the new TMPDIR takes effect and plugins respawn with sockets under `/run`.

## Heads-up

Puppet rolls a **bao restart per node** when this lands (the drop-in notifies `Service['vault']`). With auto-unseal (this cluster runs `profiles::vault::unseal`) it is a rolling non-event; if any node relies on manual unseal it will come back **sealed** and need unsealing. Merge consciously / stagger if needed.

https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT
Reviewed-on: #509
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-08-09 12:14:02 +10:00

157 lines
4.9 KiB
Puppet

# profiles::vault::server
class profiles::vault::server (
Boolean $members_lookup = false,
Variant[
String,
Undef
] $members_role = undef,
Array $vault_servers = [],
String $package_name = 'vault',
String $package_ensure = 'latest',
Boolean $disable_openbao = true,
Boolean $tls_disable = false,
Stdlib::Port $client_port = 8200,
Stdlib::Port $cluster_port = 8201,
Boolean $manage_storage_dir = false,
Stdlib::Absolutepath $data_dir = '/opt/vault',
Stdlib::Absolutepath $plugin_dir = '/opt/vault_plugins',
Stdlib::Absolutepath $bin_dir = '/usr/bin',
Stdlib::Absolutepath $ssl_crt = '/etc/pki/tls/vault/certificate.crt',
Stdlib::Absolutepath $ssl_key = '/etc/pki/tls/vault/private.key',
Stdlib::Absolutepath $ssl_ca = '/etc/pki/tls/certs/ca-bundle.crt',
Stdlib::Absolutepath $audit_log = '/var/log/vault_audit.log',
){
# set a datacentre/cluster name
$vault_cluster = "${::facts['country']}-${::facts['region']}"
# if lookup is enabled, find all the hosts in the specified role and create the servers_array
if $members_lookup and $members_role != undef {
# if it is, find hosts, sort them so they dont cause changes every run
$servers_array = sort(puppetdb_query(
"facts[certname] {
name = 'enc_role' and value = '${members_role}' and
certname in facts[certname] { name = 'region' and value = '${::facts['region']}' }
}"
).map |$fact| { $fact['certname'] })
# else use provided array from params
}else{
$servers_array = $vault_servers
}
# configure vault if servers_array isnt empty
if ! $servers_array.empty() {
# set http scheme
$http_scheme = $tls_disable ? {
true => 'http',
false => 'https'
}
# create vault urls
$server_urls = $servers_array.map |$fqdn| {
{
leader_api_addr => "${http_scheme}://${fqdn}:${client_port}",
leader_client_cert_file => $ssl_crt,
leader_client_key_file => $ssl_key,
leader_ca_cert_file => $ssl_ca,
}
}
# cleanup openbao?
if $disable_openbao {
package {'openbao':
ensure => absent,
before => Class['vault']
}
package {'openbao-vault-compat':
ensure => absent,
before => [
Class['vault'],
Package['openbao']
]
}
}
# add versionlock for package_name?
if $package_ensure != 'latest' {
yum::versionlock{$package_name:
ensure => present,
version => $package_ensure,
before => Class['vault']
}
}
class { 'vault':
package_name => $package_name,
package_ensure => $package_ensure,
manage_service => false,
manage_storage_dir => $manage_storage_dir,
enable_ui => true,
storage => {
raft => {
node_id => $::facts['networking']['fqdn'],
path => $data_dir,
retry_join => $server_urls,
}
},
api_addr => "${http_scheme}://${::facts['networking']['fqdn']}:${client_port}",
extra_config => {
cluster_addr => "${http_scheme}://${::facts['networking']['fqdn']}:${cluster_port}",
plugin_directory => $plugin_dir,
},
listener => [
{
tcp => {
address => "127.0.0.1:${client_port}",
cluster_address => "127.0.0.1:${cluster_port}",
tls_disable => true,
}
},
{
tcp => {
address => "${::facts['networking']['ip']}:${client_port}",
cluster_address => "${::facts['networking']['ip']}:${cluster_port}",
tls_disable => $tls_disable,
tls_cert_file => $ssl_crt,
tls_key_file => $ssl_key,
}
}
]
}
# ensure the vault audit log exists
file { $audit_log:
ensure => 'file',
owner => 'vault',
group => 'vault',
mode => '0600',
}
# go-plugin creates each secrets plugin's control socket under TMPDIR
# (/tmp/pluginNNN by default); systemd-tmpfiles-clean reaps aged /tmp files
# and severs the socket of a long-lived plugin, orphaning the process. Point
# TMPDIR at a per-start RuntimeDirectory on /run (tmpfs, no age cleanup).
systemd::manage_dropin { 'plugin-tmpdir.conf':
unit => 'vault.service',
service_entry => {
'RuntimeDirectory' => 'vault-plugins',
'RuntimeDirectoryMode' => '0700',
'Environment' => 'TMPDIR=/run/vault-plugins',
},
notify => Service['vault'],
}
service { 'vault':
ensure => true,
enable => true,
subscribe => [File[$ssl_crt], File[$ssl_key]],
}
# include classes to manage vault
include profiles::vault::unseal
}
}