dd38651a9f
## Why An out-of-band OpenJDK upgrade (java-17-openjdk 17.0.18 -> 17.0.19, delivered by the AlmaLinux repo migration in #496) removed the old versioned JAVA_HOME while the puppetserver JVMs kept running against the now-deleted files. The running JVM re-execs `jspawnhelper` from its original (deleted) JAVA_HOME on every posix_spawn, so `ProcessBuilder` fails with `error=2, No such file or directory`. That broke the exec ENC (`/opt/cobbler-enc/cobbler-enc`) and 500'd every catalog compile across all 6 masters, failing 136/143 nodes. The masters could not self-heal because nothing restarts the JVM after a java upgrade. Incident was resolved by manually restarting puppetserver on all 6 masters. ## Change - Add `/usr/local/bin/puppetserver_jvm_guard.sh`: restarts puppetserver if a running JVM (`puppet-server-release.jar`) is executing from deleted binaries. - Add a `puppetserver-jvm-guard` systemd timer (every 5 min, mirrors the existing generate-types timer pattern) that runs the guard, so any future JVM/library upgrade recovers automatically. https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv Reviewed-on: #499 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
198 lines
6.7 KiB
Puppet
198 lines
6.7 KiB
Puppet
# Class: profiles::puppet::server
|
|
#
|
|
# This class manages Puppet server's configuration and service.
|
|
class profiles::puppet::server (
|
|
Stdlib::Absolutepath $vardir = '/opt/puppetlabs/server/data/puppetserver',
|
|
Stdlib::Absolutepath $logdir = '/var/log/puppetlabs/puppetserver',
|
|
Stdlib::Absolutepath $rundir = '/var/run/puppetlabs/puppetserver',
|
|
Stdlib::Absolutepath $pidfile = '/var/run/puppetlabs/puppetserver/puppetserver.pid',
|
|
Stdlib::Absolutepath $codedir = '/etc/puppetlabs/code',
|
|
Array[String] $dns_alt_names = [
|
|
$facts['networking']['fqdn'],
|
|
$facts['networking']['hostname'],
|
|
],
|
|
Stdlib::Fqdn $agent_server = 'puppetmaster',
|
|
Stdlib::Fqdn $report_server = $agent_server,
|
|
Stdlib::Fqdn $ca_server = 'puppetca',
|
|
String $node_terminus = 'exec',
|
|
String $external_nodes = '/opt/cobbler-enc/cobbler-enc',
|
|
String $default_environment = 'develop',
|
|
String $environment = 'develop',
|
|
Stdlib::Absolutepath $autosign = '/etc/puppetlabs/puppet/autosign.conf',
|
|
Stdlib::Absolutepath $default_manifest = "${codedir}/environments/${default_environment}/manifests",
|
|
String $reports = 'puppetdb',
|
|
Boolean $storeconfigs = true,
|
|
String $storeconfigs_backend = 'puppetdb',
|
|
Boolean $usecacheonfailure = false,
|
|
Boolean $report = true,
|
|
Integer $runinterval = 1800,
|
|
Integer $runtimeout = 3600,
|
|
Boolean $show_diff = true,
|
|
Integer $facts_soft_limit = 4096,
|
|
) {
|
|
|
|
# add a bool for if this host is a ca, used in the auth.conf file
|
|
$is_ca = hiera('profiles::puppet::puppetca::is_puppetca', false)
|
|
|
|
file { '/etc/puppetlabs/puppet/puppet.conf':
|
|
ensure => file,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
content => epp('profiles/puppet/server/puppet.conf.epp', {
|
|
'vardir' => $vardir,
|
|
'logdir' => $logdir,
|
|
'rundir' => $rundir,
|
|
'pidfile' => $pidfile,
|
|
'codedir' => $codedir,
|
|
'dns_alt_names' => join(sort($dns_alt_names), ','),
|
|
'server' => $agent_server,
|
|
'ca_server' => $ca_server,
|
|
'environment' => $environment,
|
|
'report' => $report,
|
|
'runinterval' => $runinterval,
|
|
'runtimeout' => $runtimeout,
|
|
'show_diff' => $show_diff,
|
|
'report_server' => $report_server,
|
|
'node_terminus' => $node_terminus,
|
|
'external_nodes' => $external_nodes,
|
|
'autosign' => $autosign,
|
|
'default_manifest' => $default_manifest,
|
|
'default_environment' => $default_environment,
|
|
'storeconfigs' => $storeconfigs,
|
|
'storeconfigs_backend' => $storeconfigs_backend,
|
|
'reports' => $reports,
|
|
'usecacheonfailure' => $usecacheonfailure,
|
|
'facts_soft_limit' => $facts_soft_limit,
|
|
}),
|
|
notify => Service['puppetserver'],
|
|
}
|
|
|
|
file { '/etc/puppetlabs/puppetserver/conf.d/auth.conf':
|
|
ensure => 'file',
|
|
content => template('profiles/puppet/server/auth.conf.erb'),
|
|
group => 'root',
|
|
owner => 'root',
|
|
mode => '0644',
|
|
notify => Service['puppetserver'],
|
|
}
|
|
|
|
service { 'puppetserver':
|
|
ensure => running,
|
|
enable => true,
|
|
hasstatus => true,
|
|
hasrestart => true,
|
|
}
|
|
|
|
# generate puppet types when restarting
|
|
systemd::manage_dropin { 'generate_types.conf':
|
|
ensure => absent,
|
|
unit => 'puppetserver.service',
|
|
service_entry => {
|
|
'ExecStartPost' => [
|
|
"/opt/puppetlabs/bin/puppet generate types --environmentpath ${codedir}/environments",
|
|
],
|
|
},
|
|
}
|
|
|
|
file { '/usr/local/bin/puppet_generate_types.sh':
|
|
ensure => file,
|
|
mode => '0755',
|
|
content => @("EOF")
|
|
#!/bin/bash
|
|
/opt/puppetlabs/bin/puppet generate types --environmentpath ${codedir}/environments
|
|
exit 0
|
|
| EOF
|
|
}
|
|
|
|
$_timer = @(EOT)
|
|
[Unit]
|
|
Description=puppet-generate-types timer
|
|
[Timer]
|
|
OnCalendar=*:0/1
|
|
RandomizedDelaySec=1s
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOT
|
|
|
|
$_service = @(EOT)
|
|
[Unit]
|
|
Description=puppet-generate-types service
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/puppet_generate_types.sh
|
|
User=root
|
|
Group=root
|
|
PermissionsStartOnly=false
|
|
PrivateTmp=no
|
|
EOT
|
|
|
|
systemd::timer { 'puppet-generate-types.timer':
|
|
timer_content => $_timer,
|
|
service_content => $_service,
|
|
active => true,
|
|
enable => true,
|
|
require => File['/usr/local/bin/puppet_generate_types.sh'],
|
|
}
|
|
|
|
# Guard against an OpenJDK (or other JVM library) package upgrade landing
|
|
# underneath a long-running puppetserver. When the java package is replaced
|
|
# out-of-band (e.g. a yum repo/version bump), the old versioned JAVA_HOME is
|
|
# removed while the JVM keeps running against the now-deleted files. The JVM
|
|
# still execs jspawnhelper from its original (deleted) JAVA_HOME on every
|
|
# subprocess spawn, so ProcessBuilder fails with "error=2, No such file or
|
|
# directory" -- which breaks the exec ENC (/opt/cobbler-enc/cobbler-enc) and
|
|
# 500s every catalog compile fleet-wide. A restart re-binds to the new JVM.
|
|
# This timer detects the running JVM executing from deleted binaries and
|
|
# restarts the service to recover automatically.
|
|
file { '/usr/local/bin/puppetserver_jvm_guard.sh':
|
|
ensure => file,
|
|
mode => '0755',
|
|
content => @(EOF),
|
|
#!/bin/bash
|
|
# Restart puppetserver if its running JVM is executing deleted binaries
|
|
# (e.g. an OpenJDK package upgrade replaced the versioned JAVA_HOME),
|
|
# which breaks subprocess spawning and thus ENC / catalog compilation.
|
|
for pid in $(pgrep -f puppet-server-release.jar); do
|
|
exe=$(readlink "/proc/${pid}/exe" 2>/dev/null)
|
|
case "${exe}" in
|
|
*'(deleted)'*)
|
|
logger -t puppetserver-jvm-guard "puppetserver JVM (pid ${pid}) running on deleted binaries; restarting"
|
|
systemctl restart puppetserver
|
|
exit 0
|
|
;;
|
|
esac
|
|
done
|
|
exit 0
|
|
| EOF
|
|
}
|
|
|
|
$_guard_timer = @(EOT)
|
|
[Unit]
|
|
Description=puppetserver JVM guard timer
|
|
[Timer]
|
|
OnCalendar=*:0/5
|
|
RandomizedDelaySec=30s
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOT
|
|
|
|
$_guard_service = @(EOT)
|
|
[Unit]
|
|
Description=puppetserver JVM guard service
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/puppetserver_jvm_guard.sh
|
|
User=root
|
|
Group=root
|
|
EOT
|
|
|
|
systemd::timer { 'puppetserver-jvm-guard.timer':
|
|
timer_content => $_guard_timer,
|
|
service_content => $_guard_service,
|
|
active => true,
|
|
enable => true,
|
|
require => File['/usr/local/bin/puppetserver_jvm_guard.sh'],
|
|
}
|
|
}
|