ghp: fix first-install ordering deadlock via sync-waves

The migrate Job was a PreSync hook but connects to postgres-rw, whose CNPG
Cluster + generated postgres-app secret apply in the Sync phase (after all
PreSync hooks). On a fresh install migrate ran before Postgres existed, failed,
exhausted backoffLimit, failed PreSync, and blocked the Sync phase that creates
the DB.

- Move migrate off PreSync to a Sync-phase hook at sync-wave 1.
- Put the CNPG Cluster + backup resources, VSO auth/secrets, and Certificate at
  wave 0 so the DB is Healthy (and creds/cert exist) before migrate runs.
- Put the Deployment + Service/Gateway/HTTPRoute/PDB/VMServiceScrape at wave 2
  so serve starts after migrate completes.
- Add a writable /tmp emptyDir to the serve container and migrate Job (root FS
  is read-only) so codeload/staging writes cannot crash the process.

apps/base/ghp/certificate.yaml

@@ -9,6 +9,8 @@ metadata:
   labels:
     app.kubernetes.io/name: ghp
     app.kubernetes.io/instance: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   secretName: ghp-tls
   issuerRef:

apps/base/ghp/cnpg_backup.yaml

@@ -6,6 +6,8 @@ kind: ObjectStoreUser
 metadata:
   name: cnpg-ghp-backup
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   displayName: "CNPG backup owner (ghp)"
   uid: cnpg-ghp-backup
@@ -18,6 +20,8 @@ kind: Bucket
 metadata:
   name: cnpg-ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   placementTarget: ec
   bucketName: cnpg-ghp
@@ -35,6 +39,8 @@ kind: ScheduledBackup
 metadata:
   name: cnpg-ghp-nightly
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   schedule: "0 50 1 * * *"
   immediate: false

apps/base/ghp/cnpg_cluster.yaml

@@ -4,6 +4,10 @@ kind: Cluster
 metadata:
   name: postgres
   namespace: ghp
+  annotations:
+    # Wave 0: DB (and the generated postgres-app Secret) must be Healthy before
+    # the wave-1 migrate Job runs. ArgoCD gates on the Cluster's health status.
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   affinity:
     podAntiAffinityType: preferred

apps/base/ghp/deployment.yaml

@@ -5,6 +5,8 @@ metadata:
   name: ghp
   namespace: ghp
   annotations:
+    # Wave 2: serve only after the wave-1 migrate Job completes.
+    argocd.argoproj.io/sync-wave: "2"
     configmap.reloader.stakater.com/auto: "true"
     secret.reloader.stakater.com/reload: "ghp-github-app,ghp-app,ghp-tls,postgres-app"
 spec:
@@ -35,7 +37,7 @@ spec:
           image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0
           imagePullPolicy: IfNotPresent
           # Drop the image's default --migrate so replicas never race migrations;
-          # schema is applied by the PreSync hook Job instead.
+          # schema is applied by the wave-1 migrate hook Job instead.
           command: ["/ghp", "serve"]
           ports:
             - containerPort: 8443
@@ -93,6 +95,8 @@ spec:
             - name: tls
               mountPath: /etc/ghp/tls
               readOnly: true
+            - name: tmp
+              mountPath: /tmp
           livenessProbe:
             failureThreshold: 3
             httpGet:
@@ -133,4 +137,9 @@ spec:
         - name: tls
           secret:
             secretName: ghp-tls
+        # Writable scratch: root FS is read-only. Disk-backed (not memory medium)
+        # so codeload tarball staging doesn't count against the pod memory limit.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 2Gi
       restartPolicy: Always

apps/base/ghp/gateway.yaml

@@ -9,6 +9,7 @@ metadata:
   labels:
     traefik.io/instance: internal
   annotations:
+    argocd.argoproj.io/sync-wave: "2"
     external-dns.alpha.kubernetes.io/hostname: ghp.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
   name: ghp

apps/base/ghp/httproute.yaml

@@ -4,6 +4,8 @@ kind: HTTPRoute
 metadata:
   name: ghp-route
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   hostnames:
     - ghp.k8s.syd1.au.unkin.net

apps/base/ghp/migrate-job.yaml

@@ -2,14 +2,21 @@
 # Runs the schema migration once per sync, before the Deployment rolls, so the
 # serve replicas never race migrations. Deleted before each re-create so a new
 # image/version re-runs it.
+#
+# Sync-phase hook at wave 1 (NOT PreSync): the CNPG Cluster + generated
+# postgres-app Secret apply at wave 0 and ArgoCD waits for the Cluster to be
+# Healthy before starting wave 1, so on a fresh install Postgres exists before
+# migrate connects. (A PreSync hook would run before the Sync phase that creates
+# the DB, deadlocking the first install.)
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: ghp-migrate
   namespace: ghp
   annotations:
-    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    argocd.argoproj.io/sync-wave: "1"
 spec:
   backoffLimit: 6
   ttlSecondsAfterFinished: 600
@@ -48,6 +55,9 @@ spec:
                   key: password
             - name: GHP_DATABASE_DSN
               value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require"
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -61,3 +71,8 @@ spec:
             requests:
               cpu: 250m
               memory: 256Mi
+      volumes:
+        # Writable scratch: root FS is read-only, so give the migrator a /tmp.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi

apps/base/ghp/pdb.yaml

@@ -4,6 +4,8 @@ kind: PodDisruptionBudget
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   minAvailable: 1
   selector:

apps/base/ghp/service.yaml

@@ -6,6 +6,8 @@ metadata:
   namespace: ghp
   labels:
     app: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   internalTrafficPolicy: Cluster
   ports:

apps/base/ghp/vaultauth.yaml

@@ -4,6 +4,8 @@ kind: VaultAuth
 metadata:
   name: default
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   allowedNamespaces:
     - ghp

apps/base/ghp/vaultstaticsecret.yaml

@@ -6,6 +6,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-github-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true
@@ -24,6 +26,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true

apps/base/ghp/vmservicescrape.yaml

@@ -7,6 +7,8 @@ kind: VMServiceScrape
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   selector:
     matchLabels: