From 5d1cc10588babf7054be2c8559c14bf0dd8cea5c Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sat, 8 Aug 2026 20:02:38 +1000
Subject: [PATCH] Pin puppet master to a single Recreate replica (#341)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Why

The puppet MASTER is the singleton CA/master. A second master, even transiently during a rolling update, races on CA/cert signing and shared state (the CA lives on a shared PVC mounted by every master pod). The master was previously driven by an HPA with `minReplicas: 2`, `maxReplicas: 5` and a `RollingUpdate` strategy, so 2-5 masters could coexist normally and a rollout would briefly run old+new masters against the same CA data — a latent CA-corruption/split-brain bug. Recreate guarantees the old pod terminates before the new one starts, so two masters never coexist.

## Changes

- Set `puppetserver-master` `spec.replicas: 1` and `spec.strategy.type: Recreate` (drops RollingUpdate).
- Remove the `puppetserver-masters-autoscaler` HPA and its kustomization entry, which forced 2-5 master replicas and would otherwise override `replicas: 1`.
- Refresh the `puppetserver-master-vpa` note to reflect the pinned-singleton, no-HPA state (VPA stays `updateMode: Off`, recommendation-only).

The compiler (`puppetserver-compiler`) remains the horizontally-scalable tier with its own HPA — untouched. puppetdb/puppetboard untouched.

https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT
---------

Co-authored-by: Ben Vincent <neotheo@gmail.com>
Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/341
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
---
 .../deployment_puppetserver-master.yaml       |  3 +-
 ...caler_puppetserver-masters-autoscaler.yaml | 37 -------------------
 apps/base/puppet/kustomization.yaml           |  1 -
 apps/base/puppet/vpa.yaml                     |  6 +--
 4 files changed, 5 insertions(+), 42 deletions(-)
 delete mode 100644 apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml

diff --git a/apps/base/puppet/deployment_puppetserver-master.yaml b/apps/base/puppet/deployment_puppetserver-master.yaml
index 54aa4e3..3f13268 100644
--- a/apps/base/puppet/deployment_puppetserver-master.yaml
+++ b/apps/base/puppet/deployment_puppetserver-master.yaml
@@ -12,12 +12,13 @@ metadata:
   name: puppetserver-master
   namespace: puppet
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/component: puppetserver
       app.kubernetes.io/name: puppetserver
   strategy:
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
diff --git a/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml b/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
deleted file mode 100644
index 028b2ef..0000000
--- a/apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-apiVersion: autoscaling/v2
-kind: HorizontalPodAutoscaler
-metadata:
-  labels:
-    app.kubernetes.io/component: puppetserver
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetserver-masters-autoscaler
-  namespace: puppet
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: puppetserver-master
-  minReplicas: 2
-  maxReplicas: 5
-  metrics:
-  - resource:
-      name: cpu
-      target:
-        averageUtilization: 75
-        type: Utilization
-    type: Resource
-  behavior:
-    scaleUp:
-      stabilizationWindowSeconds: 60
-      policies:
-      - type: Percent
-        value: 50
-        periodSeconds: 15
-    scaleDown:
-      stabilizationWindowSeconds: 300
-      policies:
-      - type: Percent
-        value: 25
-        periodSeconds: 60
diff --git a/apps/base/puppet/kustomization.yaml b/apps/base/puppet/kustomization.yaml
index bb889ad..be51457 100644
--- a/apps/base/puppet/kustomization.yaml
+++ b/apps/base/puppet/kustomization.yaml
@@ -23,7 +23,6 @@ resources:
   - deployment_puppetdb.yaml
   - deployment_puppetserver-master.yaml
   - horizontalpodautoscaler_puppetserver-compilers-autoscaler.yaml
-  - horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetboard-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetdb-autoscaler.yaml
   - gateway_puppetboard.yaml
diff --git a/apps/base/puppet/vpa.yaml b/apps/base/puppet/vpa.yaml
index 90107c6..b5165f9 100644
--- a/apps/base/puppet/vpa.yaml
+++ b/apps/base/puppet/vpa.yaml
@@ -52,9 +52,9 @@ kind: VerticalPodAutoscaler
 metadata:
   name: puppetserver-master-vpa
   namespace: puppet
-  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
-  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
-  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+  # NOTE: the master is a pinned single replica (Recreate, no HPA) so the CA/master
+  # never coexists. updateMode Off keeps this recommendation-only; do not flip to
+  # Auto/Initial, which would evict and briefly recreate the singleton pod.
 spec:
   targetRef:
     apiVersion: apps/v1