From a5f3b18e863f3247855ac5ea6c0acfe1f3b8c111 Mon Sep 17 00:00:00 2001
From: Ben Vincent <neotheo@gmail.com>
Date: Sat, 8 Aug 2026 19:58:42 +1000
Subject: [PATCH] Pin puppet master to a single Recreate replica

The puppet MASTER is the singleton CA/master: a second master, even
transiently during a rolling update, races on CA/cert signing and shared
state. It was previously driven by an HPA (minReplicas 2, maxReplicas 5)
with a RollingUpdate strategy, so 2-5 masters could coexist and a rollout
would briefly run old+new masters against the same shared CA PVC.

- Set puppetserver-master replicas to 1 and strategy to Recreate so the
  old master always terminates before a new one starts.
- Remove the puppetserver-masters-autoscaler HPA (and its kustomization
  entry) that forced 2-5 master replicas.
- Refresh the master VPA note to reflect the pinned-singleton, no-HPA state.

Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT
---
 .../deployment_puppetserver-master.yaml       |  3 +-
 ...caler_puppetserver-masters-autoscaler.yaml | 37 -------------------
 apps/base/puppet/kustomization.yaml           |  1 -
 apps/base/puppet/vpa.yaml                     |  6 +--
 4 files changed, 5 insertions(+), 42 deletions(-)
 delete mode 100644 apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml

diff --git a/apps/base/puppet/deployment_puppetserver-master.yaml b/apps/base/puppet/deployment_puppetserver-master.yaml
index 54aa4e3..3f13268 100644
--- a/apps/base/puppet/deployment_puppetserver-master.yaml
+++ b/apps/base/puppet/deployment_puppetserver-master.yaml
@@ -12,12 +12,13 @@ metadata:
   name: puppetserver-master
   namespace: puppet
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/component: puppetserver
       app.kubernetes.io/name: puppetserver
   strategy:
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
diff --git a/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml b/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
deleted file mode 100644
index 028b2ef..0000000
--- a/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-apiVersion: autoscaling/v2
-kind: HorizontalPodAutoscaler
-metadata:
-  labels:
-    app.kubernetes.io/component: puppetserver
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetserver-masters-autoscaler
-  namespace: puppet
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: puppetserver-master
-  minReplicas: 2
-  maxReplicas: 5
-  metrics:
-  - resource:
-      name: cpu
-      target:
-        averageUtilization: 75
-        type: Utilization
-    type: Resource
-  behavior:
-    scaleUp:
-      stabilizationWindowSeconds: 60
-      policies:
-      - type: Percent
-        value: 50
-        periodSeconds: 15
-    scaleDown:
-      stabilizationWindowSeconds: 300
-      policies:
-      - type: Percent
-        value: 25
-        periodSeconds: 60
diff --git a/apps/base/puppet/kustomization.yaml b/apps/base/puppet/kustomization.yaml
index bb889ad..be51457 100644
--- a/apps/base/puppet/kustomization.yaml
+++ b/apps/base/puppet/kustomization.yaml
@@ -23,7 +23,6 @@ resources:
   - deployment_puppetdb.yaml
   - deployment_puppetserver-master.yaml
   - horizontalpodautoscaler_puppetserver-compilers-autoscaler.yaml
-  - horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetboard-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetdb-autoscaler.yaml
   - gateway_puppetboard.yaml
diff --git a/apps/base/puppet/vpa.yaml b/apps/base/puppet/vpa.yaml
index 90107c6..b5165f9 100644
--- a/apps/base/puppet/vpa.yaml
+++ b/apps/base/puppet/vpa.yaml
@@ -52,9 +52,9 @@ kind: VerticalPodAutoscaler
 metadata:
   name: puppetserver-master-vpa
   namespace: puppet
-  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
-  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
-  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+  # NOTE: the master is a pinned single replica (Recreate, no HPA) so the CA/master
+  # never coexists. updateMode Off keeps this recommendation-only; do not flip to
+  # Auto/Initial, which would evict and briefly recreate the singleton pod.
 spec:
   targetRef:
     apiVersion: apps/v1
-- 
2.47.3