Files
puppet-prod/site/profiles/manifests/puppet/server.pp
T
unkinben dd38651a9f puppetserver: auto-restart JVM when its binaries are replaced by an upgrade (#499)
## Why

An out-of-band OpenJDK upgrade (java-17-openjdk 17.0.18 -> 17.0.19, delivered by the AlmaLinux repo migration in #496) removed the old versioned JAVA_HOME while the puppetserver JVMs kept running against the now-deleted files. The running JVM re-execs `jspawnhelper` from its original (deleted) JAVA_HOME on every posix_spawn, so `ProcessBuilder` fails with `error=2, No such file or directory`. That broke the exec ENC (`/opt/cobbler-enc/cobbler-enc`) and 500'd every catalog compile across all 6 masters, failing 136/143 nodes. The masters could not self-heal because nothing restarts the JVM after a java upgrade. Incident was resolved by manually restarting puppetserver on all 6 masters.

## Change

- Add `/usr/local/bin/puppetserver_jvm_guard.sh`: restarts puppetserver if a running JVM (`puppet-server-release.jar`) is executing from deleted binaries.
- Add a `puppetserver-jvm-guard` systemd timer (every 5 min, mirrors the existing generate-types timer pattern) that runs the guard, so any future JVM/library upgrade recovers automatically.

https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
Reviewed-on: #499
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-07-30 20:50:30 +10:00

198 lines
6.7 KiB
Puppet

# Class: profiles::puppet::server
#
# This class manages Puppet server's configuration and service.
class profiles::puppet::server (
Stdlib::Absolutepath $vardir = '/opt/puppetlabs/server/data/puppetserver',
Stdlib::Absolutepath $logdir = '/var/log/puppetlabs/puppetserver',
Stdlib::Absolutepath $rundir = '/var/run/puppetlabs/puppetserver',
Stdlib::Absolutepath $pidfile = '/var/run/puppetlabs/puppetserver/puppetserver.pid',
Stdlib::Absolutepath $codedir = '/etc/puppetlabs/code',
Array[String] $dns_alt_names = [
$facts['networking']['fqdn'],
$facts['networking']['hostname'],
],
Stdlib::Fqdn $agent_server = 'puppetmaster',
Stdlib::Fqdn $report_server = $agent_server,
Stdlib::Fqdn $ca_server = 'puppetca',
String $node_terminus = 'exec',
String $external_nodes = '/opt/cobbler-enc/cobbler-enc',
String $default_environment = 'develop',
String $environment = 'develop',
Stdlib::Absolutepath $autosign = '/etc/puppetlabs/puppet/autosign.conf',
Stdlib::Absolutepath $default_manifest = "${codedir}/environments/${default_environment}/manifests",
String $reports = 'puppetdb',
Boolean $storeconfigs = true,
String $storeconfigs_backend = 'puppetdb',
Boolean $usecacheonfailure = false,
Boolean $report = true,
Integer $runinterval = 1800,
Integer $runtimeout = 3600,
Boolean $show_diff = true,
Integer $facts_soft_limit = 4096,
) {
# add a bool for if this host is a ca, used in the auth.conf file
$is_ca = hiera('profiles::puppet::puppetca::is_puppetca', false)
file { '/etc/puppetlabs/puppet/puppet.conf':
ensure => file,
owner => 'root',
group => 'root',
mode => '0644',
content => epp('profiles/puppet/server/puppet.conf.epp', {
'vardir' => $vardir,
'logdir' => $logdir,
'rundir' => $rundir,
'pidfile' => $pidfile,
'codedir' => $codedir,
'dns_alt_names' => join(sort($dns_alt_names), ','),
'server' => $agent_server,
'ca_server' => $ca_server,
'environment' => $environment,
'report' => $report,
'runinterval' => $runinterval,
'runtimeout' => $runtimeout,
'show_diff' => $show_diff,
'report_server' => $report_server,
'node_terminus' => $node_terminus,
'external_nodes' => $external_nodes,
'autosign' => $autosign,
'default_manifest' => $default_manifest,
'default_environment' => $default_environment,
'storeconfigs' => $storeconfigs,
'storeconfigs_backend' => $storeconfigs_backend,
'reports' => $reports,
'usecacheonfailure' => $usecacheonfailure,
'facts_soft_limit' => $facts_soft_limit,
}),
notify => Service['puppetserver'],
}
file { '/etc/puppetlabs/puppetserver/conf.d/auth.conf':
ensure => 'file',
content => template('profiles/puppet/server/auth.conf.erb'),
group => 'root',
owner => 'root',
mode => '0644',
notify => Service['puppetserver'],
}
service { 'puppetserver':
ensure => running,
enable => true,
hasstatus => true,
hasrestart => true,
}
# generate puppet types when restarting
systemd::manage_dropin { 'generate_types.conf':
ensure => absent,
unit => 'puppetserver.service',
service_entry => {
'ExecStartPost' => [
"/opt/puppetlabs/bin/puppet generate types --environmentpath ${codedir}/environments",
],
},
}
file { '/usr/local/bin/puppet_generate_types.sh':
ensure => file,
mode => '0755',
content => @("EOF")
#!/bin/bash
/opt/puppetlabs/bin/puppet generate types --environmentpath ${codedir}/environments
exit 0
| EOF
}
$_timer = @(EOT)
[Unit]
Description=puppet-generate-types timer
[Timer]
OnCalendar=*:0/1
RandomizedDelaySec=1s
[Install]
WantedBy=timers.target
EOT
$_service = @(EOT)
[Unit]
Description=puppet-generate-types service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/puppet_generate_types.sh
User=root
Group=root
PermissionsStartOnly=false
PrivateTmp=no
EOT
systemd::timer { 'puppet-generate-types.timer':
timer_content => $_timer,
service_content => $_service,
active => true,
enable => true,
require => File['/usr/local/bin/puppet_generate_types.sh'],
}
# Guard against an OpenJDK (or other JVM library) package upgrade landing
# underneath a long-running puppetserver. When the java package is replaced
# out-of-band (e.g. a yum repo/version bump), the old versioned JAVA_HOME is
# removed while the JVM keeps running against the now-deleted files. The JVM
# still execs jspawnhelper from its original (deleted) JAVA_HOME on every
# subprocess spawn, so ProcessBuilder fails with "error=2, No such file or
# directory" -- which breaks the exec ENC (/opt/cobbler-enc/cobbler-enc) and
# 500s every catalog compile fleet-wide. A restart re-binds to the new JVM.
# This timer detects the running JVM executing from deleted binaries and
# restarts the service to recover automatically.
file { '/usr/local/bin/puppetserver_jvm_guard.sh':
ensure => file,
mode => '0755',
content => @(EOF),
#!/bin/bash
# Restart puppetserver if its running JVM is executing deleted binaries
# (e.g. an OpenJDK package upgrade replaced the versioned JAVA_HOME),
# which breaks subprocess spawning and thus ENC / catalog compilation.
for pid in $(pgrep -f puppet-server-release.jar); do
exe=$(readlink "/proc/${pid}/exe" 2>/dev/null)
case "${exe}" in
*'(deleted)'*)
logger -t puppetserver-jvm-guard "puppetserver JVM (pid ${pid}) running on deleted binaries; restarting"
systemctl restart puppetserver
exit 0
;;
esac
done
exit 0
| EOF
}
$_guard_timer = @(EOT)
[Unit]
Description=puppetserver JVM guard timer
[Timer]
OnCalendar=*:0/5
RandomizedDelaySec=30s
[Install]
WantedBy=timers.target
EOT
$_guard_service = @(EOT)
[Unit]
Description=puppetserver JVM guard service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/puppetserver_jvm_guard.sh
User=root
Group=root
EOT
systemd::timer { 'puppetserver-jvm-guard.timer':
timer_content => $_guard_timer,
service_content => $_guard_service,
active => true,
enable => true,
require => File['/usr/local/bin/puppetserver_jvm_guard.sh'],
}
}