diff --git a/site/profiles/manifests/puppet/server.pp b/site/profiles/manifests/puppet/server.pp index fe1ae6f..dd4cb3a 100644 --- a/site/profiles/manifests/puppet/server.pp +++ b/site/profiles/manifests/puppet/server.pp @@ -134,4 +134,64 @@ class profiles::puppet::server ( enable => true, require => File['/usr/local/bin/puppet_generate_types.sh'], } + + # Guard against an OpenJDK (or other JVM library) package upgrade landing + # underneath a long-running puppetserver. When the java package is replaced + # out-of-band (e.g. a yum repo/version bump), the old versioned JAVA_HOME is + # removed while the JVM keeps running against the now-deleted files. The JVM + # still execs jspawnhelper from its original (deleted) JAVA_HOME on every + # subprocess spawn, so ProcessBuilder fails with "error=2, No such file or + # directory" -- which breaks the exec ENC (/opt/cobbler-enc/cobbler-enc) and + # 500s every catalog compile fleet-wide. A restart re-binds to the new JVM. + # This timer detects the running JVM executing from deleted binaries and + # restarts the service to recover automatically. + file { '/usr/local/bin/puppetserver_jvm_guard.sh': + ensure => file, + mode => '0755', + content => @(EOF), + #!/bin/bash + # Restart puppetserver if its running JVM is executing deleted binaries + # (e.g. an OpenJDK package upgrade replaced the versioned JAVA_HOME), + # which breaks subprocess spawning and thus ENC / catalog compilation. + for pid in $(pgrep -f puppet-server-release.jar); do + exe=$(readlink "/proc/${pid}/exe" 2>/dev/null) + case "${exe}" in + *'(deleted)'*) + logger -t puppetserver-jvm-guard "puppetserver JVM (pid ${pid}) running on deleted binaries; restarting" + systemctl restart puppetserver + exit 0 + ;; + esac + done + exit 0 + | EOF + } + + $_guard_timer = @(EOT) + [Unit] + Description=puppetserver JVM guard timer + [Timer] + OnCalendar=*:0/5 + RandomizedDelaySec=30s + [Install] + WantedBy=timers.target + EOT + + $_guard_service = @(EOT) + [Unit] + Description=puppetserver JVM guard service + [Service] + Type=oneshot + ExecStart=/usr/local/bin/puppetserver_jvm_guard.sh + User=root + Group=root + EOT + + systemd::timer { 'puppetserver-jvm-guard.timer': + timer_content => $_guard_timer, + service_content => $_guard_service, + active => true, + enable => true, + require => File['/usr/local/bin/puppetserver_jvm_guard.sh'], + } }