From 42c1794d0ec6ae7875d2690e9d0ef20ee63d66c9 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Thu, 13 Aug 2026 19:53:09 +1000
Subject: [PATCH] ghp: fix first-install ordering deadlock via sync-waves

The migrate Job was a PreSync hook but connects to postgres-rw, whose CNPG
Cluster + generated postgres-app secret apply in the Sync phase (after all
PreSync hooks). On a fresh install migrate ran before Postgres existed, failed,
exhausted backoffLimit, failed PreSync, and blocked the Sync phase that creates
the DB.

- Move migrate off PreSync to a Sync-phase hook at sync-wave 1.
- Put the CNPG Cluster + backup resources, VSO auth/secrets, and Certificate at
  wave 0 so the DB is Healthy (and creds/cert exist) before migrate runs.
- Put the Deployment + Service/Gateway/HTTPRoute/PDB/VMServiceScrape at wave 2
  so serve starts after migrate completes.
- Add a writable /tmp emptyDir to the serve container and migrate Job (root FS
  is read-only) so codeload/staging writes cannot crash the process.
---
 apps/base/ghp/certificate.yaml       |  2 ++
 apps/base/ghp/cnpg_backup.yaml       |  6 ++++++
 apps/base/ghp/cnpg_cluster.yaml      |  4 ++++
 apps/base/ghp/deployment.yaml        | 11 ++++++++++-
 apps/base/ghp/gateway.yaml           |  1 +
 apps/base/ghp/httproute.yaml         |  2 ++
 apps/base/ghp/migrate-job.yaml       | 17 ++++++++++++++++-
 apps/base/ghp/pdb.yaml               |  2 ++
 apps/base/ghp/service.yaml           |  2 ++
 apps/base/ghp/vaultauth.yaml         |  2 ++
 apps/base/ghp/vaultstaticsecret.yaml |  4 ++++
 apps/base/ghp/vmservicescrape.yaml   |  2 ++
 12 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/apps/base/ghp/certificate.yaml b/apps/base/ghp/certificate.yaml
index 893c396..3e644ae 100644
--- a/apps/base/ghp/certificate.yaml
+++ b/apps/base/ghp/certificate.yaml
@@ -9,6 +9,8 @@ metadata:
   labels:
     app.kubernetes.io/name: ghp
     app.kubernetes.io/instance: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   secretName: ghp-tls
   issuerRef:
diff --git a/apps/base/ghp/cnpg_backup.yaml b/apps/base/ghp/cnpg_backup.yaml
index 323b579..c0df5d6 100644
--- a/apps/base/ghp/cnpg_backup.yaml
+++ b/apps/base/ghp/cnpg_backup.yaml
@@ -6,6 +6,8 @@ kind: ObjectStoreUser
 metadata:
   name: cnpg-ghp-backup
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   displayName: "CNPG backup owner (ghp)"
   uid: cnpg-ghp-backup
@@ -18,6 +20,8 @@ kind: Bucket
 metadata:
   name: cnpg-ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   placementTarget: ec
   bucketName: cnpg-ghp
@@ -35,6 +39,8 @@ kind: ScheduledBackup
 metadata:
   name: cnpg-ghp-nightly
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   schedule: "0 50 1 * * *"
   immediate: false
diff --git a/apps/base/ghp/cnpg_cluster.yaml b/apps/base/ghp/cnpg_cluster.yaml
index bf16319..76a01d0 100644
--- a/apps/base/ghp/cnpg_cluster.yaml
+++ b/apps/base/ghp/cnpg_cluster.yaml
@@ -4,6 +4,10 @@ kind: Cluster
 metadata:
   name: postgres
   namespace: ghp
+  annotations:
+    # Wave 0: DB (and the generated postgres-app Secret) must be Healthy before
+    # the wave-1 migrate Job runs. ArgoCD gates on the Cluster's health status.
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   affinity:
     podAntiAffinityType: preferred
diff --git a/apps/base/ghp/deployment.yaml b/apps/base/ghp/deployment.yaml
index 94787c1..a9204fe 100644
--- a/apps/base/ghp/deployment.yaml
+++ b/apps/base/ghp/deployment.yaml
@@ -5,6 +5,8 @@ metadata:
   name: ghp
   namespace: ghp
   annotations:
+    # Wave 2: serve only after the wave-1 migrate Job completes.
+    argocd.argoproj.io/sync-wave: "2"
     configmap.reloader.stakater.com/auto: "true"
     secret.reloader.stakater.com/reload: "ghp-github-app,ghp-app,ghp-tls,postgres-app"
 spec:
@@ -35,7 +37,7 @@ spec:
           image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0
           imagePullPolicy: IfNotPresent
           # Drop the image's default --migrate so replicas never race migrations;
-          # schema is applied by the PreSync hook Job instead.
+          # schema is applied by the wave-1 migrate hook Job instead.
           command: ["/ghp", "serve"]
           ports:
             - containerPort: 8443
@@ -93,6 +95,8 @@ spec:
             - name: tls
               mountPath: /etc/ghp/tls
               readOnly: true
+            - name: tmp
+              mountPath: /tmp
           livenessProbe:
             failureThreshold: 3
             httpGet:
@@ -133,4 +137,9 @@ spec:
         - name: tls
           secret:
             secretName: ghp-tls
+        # Writable scratch: root FS is read-only. Disk-backed (not memory medium)
+        # so codeload tarball staging doesn't count against the pod memory limit.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 2Gi
       restartPolicy: Always
diff --git a/apps/base/ghp/gateway.yaml b/apps/base/ghp/gateway.yaml
index a61ad4b..80fb011 100644
--- a/apps/base/ghp/gateway.yaml
+++ b/apps/base/ghp/gateway.yaml
@@ -9,6 +9,7 @@ metadata:
   labels:
     traefik.io/instance: internal
   annotations:
+    argocd.argoproj.io/sync-wave: "2"
     external-dns.alpha.kubernetes.io/hostname: ghp.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
   name: ghp
diff --git a/apps/base/ghp/httproute.yaml b/apps/base/ghp/httproute.yaml
index f692330..af65a04 100644
--- a/apps/base/ghp/httproute.yaml
+++ b/apps/base/ghp/httproute.yaml
@@ -4,6 +4,8 @@ kind: HTTPRoute
 metadata:
   name: ghp-route
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   hostnames:
     - ghp.k8s.syd1.au.unkin.net
diff --git a/apps/base/ghp/migrate-job.yaml b/apps/base/ghp/migrate-job.yaml
index 6e10b5e..b067d12 100644
--- a/apps/base/ghp/migrate-job.yaml
+++ b/apps/base/ghp/migrate-job.yaml
@@ -2,14 +2,21 @@
 # Runs the schema migration once per sync, before the Deployment rolls, so the
 # serve replicas never race migrations. Deleted before each re-create so a new
 # image/version re-runs it.
+#
+# Sync-phase hook at wave 1 (NOT PreSync): the CNPG Cluster + generated
+# postgres-app Secret apply at wave 0 and ArgoCD waits for the Cluster to be
+# Healthy before starting wave 1, so on a fresh install Postgres exists before
+# migrate connects. (A PreSync hook would run before the Sync phase that creates
+# the DB, deadlocking the first install.)
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: ghp-migrate
   namespace: ghp
   annotations:
-    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    argocd.argoproj.io/sync-wave: "1"
 spec:
   backoffLimit: 6
   ttlSecondsAfterFinished: 600
@@ -48,6 +55,9 @@ spec:
                   key: password
             - name: GHP_DATABASE_DSN
               value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require"
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -61,3 +71,8 @@ spec:
             requests:
               cpu: 250m
               memory: 256Mi
+      volumes:
+        # Writable scratch: root FS is read-only, so give the migrator a /tmp.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi
diff --git a/apps/base/ghp/pdb.yaml b/apps/base/ghp/pdb.yaml
index 5953bf0..4a47f6d 100644
--- a/apps/base/ghp/pdb.yaml
+++ b/apps/base/ghp/pdb.yaml
@@ -4,6 +4,8 @@ kind: PodDisruptionBudget
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   minAvailable: 1
   selector:
diff --git a/apps/base/ghp/service.yaml b/apps/base/ghp/service.yaml
index f1519b9..7c3e1d3 100644
--- a/apps/base/ghp/service.yaml
+++ b/apps/base/ghp/service.yaml
@@ -6,6 +6,8 @@ metadata:
   namespace: ghp
   labels:
     app: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   internalTrafficPolicy: Cluster
   ports:
diff --git a/apps/base/ghp/vaultauth.yaml b/apps/base/ghp/vaultauth.yaml
index cfbd3d4..1678ba0 100644
--- a/apps/base/ghp/vaultauth.yaml
+++ b/apps/base/ghp/vaultauth.yaml
@@ -4,6 +4,8 @@ kind: VaultAuth
 metadata:
   name: default
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   allowedNamespaces:
     - ghp
diff --git a/apps/base/ghp/vaultstaticsecret.yaml b/apps/base/ghp/vaultstaticsecret.yaml
index c01961b..a44e926 100644
--- a/apps/base/ghp/vaultstaticsecret.yaml
+++ b/apps/base/ghp/vaultstaticsecret.yaml
@@ -6,6 +6,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-github-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true
@@ -24,6 +26,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true
diff --git a/apps/base/ghp/vmservicescrape.yaml b/apps/base/ghp/vmservicescrape.yaml
index 5923764..a91bde9 100644
--- a/apps/base/ghp/vmservicescrape.yaml
+++ b/apps/base/ghp/vmservicescrape.yaml
@@ -7,6 +7,8 @@ kind: VMServiceScrape
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   selector:
     matchLabels: