From 1d5e8edbdd838b68144dc453396afcd95cc50058 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Thu, 13 Aug 2026 19:40:31 +1000
Subject: [PATCH 1/3] add ghp (GitHub proxy) deployment

Deploy goodtune/ghp so estate agents have a GitHub proxy. Mirrors the
artifactapi app structure under apps/base/ghp with a CNPG Postgres backend,
VSO-synced GitHub App + encryption secrets, an internal Vault-PKI serving cert,
a PreSync migration hook, and platform ApplicationSet/project registration.
---
 apps/base/ghp/certificate.yaml               |  26 ++++
 apps/base/ghp/cnpg_backup.yaml               |  44 ++++++
 apps/base/ghp/cnpg_cluster.yaml              | 113 +++++++++++++++
 apps/base/ghp/configmap.yaml                 |  22 +++
 apps/base/ghp/deployment.yaml                | 136 +++++++++++++++++++
 apps/base/ghp/gateway.yaml                   |  38 ++++++
 apps/base/ghp/httproute.yaml                 |  29 ++++
 apps/base/ghp/kustomization.yaml             |  20 +++
 apps/base/ghp/migrate-job.yaml               |  63 +++++++++
 apps/base/ghp/namespace.yaml                 |   5 +
 apps/base/ghp/pdb.yaml                       |  11 ++
 apps/base/ghp/service.yaml                   |  27 ++++
 apps/base/ghp/serviceaccount.yaml            |   8 ++
 apps/base/ghp/vaultauth.yaml                 |  18 +++
 apps/base/ghp/vaultstaticsecret.yaml         |  37 +++++
 apps/base/ghp/vmservicescrape.yaml           |  16 +++
 apps/overlays/au-syd1/ghp/kustomization.yaml |   6 +
 argocd/applicationsets/platform.yaml         |   1 +
 argocd/projects/platform.yaml                |   2 +
 19 files changed, 622 insertions(+)
 create mode 100644 apps/base/ghp/certificate.yaml
 create mode 100644 apps/base/ghp/cnpg_backup.yaml
 create mode 100644 apps/base/ghp/cnpg_cluster.yaml
 create mode 100644 apps/base/ghp/configmap.yaml
 create mode 100644 apps/base/ghp/deployment.yaml
 create mode 100644 apps/base/ghp/gateway.yaml
 create mode 100644 apps/base/ghp/httproute.yaml
 create mode 100644 apps/base/ghp/kustomization.yaml
 create mode 100644 apps/base/ghp/migrate-job.yaml
 create mode 100644 apps/base/ghp/namespace.yaml
 create mode 100644 apps/base/ghp/pdb.yaml
 create mode 100644 apps/base/ghp/service.yaml
 create mode 100644 apps/base/ghp/serviceaccount.yaml
 create mode 100644 apps/base/ghp/vaultauth.yaml
 create mode 100644 apps/base/ghp/vaultstaticsecret.yaml
 create mode 100644 apps/base/ghp/vmservicescrape.yaml
 create mode 100644 apps/overlays/au-syd1/ghp/kustomization.yaml

diff --git a/apps/base/ghp/certificate.yaml b/apps/base/ghp/certificate.yaml
new file mode 100644
index 0000000..893c396
--- /dev/null
+++ b/apps/base/ghp/certificate.yaml
@@ -0,0 +1,26 @@
+---
+# Serving cert off the internal Vault-PKI CA (agents already trust vault-ca-cert).
+# SANs cover the GitHub endpoints ghp impersonates plus its own management host.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ghp-tls
+  namespace: ghp
+  labels:
+    app.kubernetes.io/name: ghp
+    app.kubernetes.io/instance: ghp
+spec:
+  secretName: ghp-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: vault-issuer
+  commonName: ghp.k8s.syd1.au.unkin.net
+  dnsNames:
+    - github.com
+    - api.github.com
+    - codeload.github.com
+    - "*.githubcopilot.com"
+    - ghp.k8s.syd1.au.unkin.net
+  privateKey:
+    algorithm: RSA
+    size: 4096
diff --git a/apps/base/ghp/cnpg_backup.yaml b/apps/base/ghp/cnpg_backup.yaml
new file mode 100644
index 0000000..323b579
--- /dev/null
+++ b/apps/base/ghp/cnpg_backup.yaml
@@ -0,0 +1,44 @@
+---
+# Ceph RGW (S3) backup target for the ghp CNPG cluster, provisioned by the
+# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-ghp-backup
+  namespace: ghp
+spec:
+  displayName: "CNPG backup owner (ghp)"
+  uid: cnpg-ghp-backup
+  maxBuckets: 5
+  secretName: cnpg-ghp-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-ghp
+  namespace: ghp
+spec:
+  placementTarget: ec
+  bucketName: cnpg-ghp
+  ownerRef: cnpg-ghp-backup
+  versioning: false
+  tags:
+    app: ghp
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+# Nightly base backup; continuous WAL archiving is always-on via the Cluster's
+# spec.backup.barmanObjectStore. Staggered off other clusters' schedules.
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-ghp-nightly
+  namespace: ghp
+spec:
+  schedule: "0 50 1 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: postgres
diff --git a/apps/base/ghp/cnpg_cluster.yaml b/apps/base/ghp/cnpg_cluster.yaml
new file mode 100644
index 0000000..bf16319
--- /dev/null
+++ b/apps/base/ghp/cnpg_cluster.yaml
@@ -0,0 +1,113 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgres
+  namespace: ghp
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-ghp
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-ghp-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-ghp-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: ghp
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      # No secret ref: CNPG mints the owner credentials and publishes them in the
+      # generated "postgres-app" Secret, which the Deployment reads to build the DSN.
+      database: ghp
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: ghp
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: 500m
+      memory: 512Mi
+    requests:
+      cpu: 50m
+      memory: 256Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 20Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600
diff --git a/apps/base/ghp/configmap.yaml b/apps/base/ghp/configmap.yaml
new file mode 100644
index 0000000..f35f4ed
--- /dev/null
+++ b/apps/base/ghp/configmap.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ghp-env
+  namespace: ghp
+data:
+  GHP_DATABASE_DRIVER: postgres
+  # nonroot cannot bind <1024; listen high and remap in the Service.
+  GHP_SERVER_HTTPS_LISTEN: ":8443"
+  GHP_SERVER_HTTP_LISTEN: ":8080"
+  GHP_METRICS_LISTEN: ":9136"
+  GHP_METRICS_ENABLED: "true"
+  GHP_SERVER_BASE_URL: https://ghp.k8s.syd1.au.unkin.net
+  GHP_SERVER_MANAGEMENT_HOST: ghp.k8s.syd1.au.unkin.net
+  # private_key key from the ghp-github-app Secret, mounted as a file.
+  GHP_GITHUB_PRIVATE_KEY_FILE: /etc/ghp/github-app/private_key
+  # cert-manager Certificate ghp-tls, mounted from the ghp-tls Secret.
+  GHP_TLS_CERT_FILE: /etc/ghp/tls/tls.crt
+  GHP_TLS_KEY_FILE: /etc/ghp/tls/tls.key
+  # PLACEHOLDER: set to Ben's GitHub username before ghp will admit an admin.
+  GHP_ADMINS: "REPLACE_ME_ben_github_username"
diff --git a/apps/base/ghp/deployment.yaml b/apps/base/ghp/deployment.yaml
new file mode 100644
index 0000000..94787c1
--- /dev/null
+++ b/apps/base/ghp/deployment.yaml
@@ -0,0 +1,136 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ghp
+  namespace: ghp
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "ghp-github-app,ghp-app,ghp-tls,postgres-app"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ghp
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: ghp
+    spec:
+      serviceAccountName: ghp
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: ghp
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0
+          imagePullPolicy: IfNotPresent
+          # Drop the image's default --migrate so replicas never race migrations;
+          # schema is applied by the PreSync hook Job instead.
+          command: ["/ghp", "serve"]
+          ports:
+            - containerPort: 8443
+              name: https
+              protocol: TCP
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+            - containerPort: 9136
+              name: metrics
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: ghp-env
+                optional: false
+          env:
+            # DSN assembled from the CNPG-generated postgres-app Secret; $(VAR)
+            # expansion resolves the two env entries defined above it.
+            - name: GHP_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-app
+                  key: username
+            - name: GHP_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-app
+                  key: password
+            - name: GHP_DATABASE_DSN
+              value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require"
+            - name: GHP_GITHUB_APP_ID
+              valueFrom:
+                secretKeyRef:
+                  name: ghp-github-app
+                  key: app_id
+            - name: GHP_GITHUB_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: ghp-github-app
+                  key: client_id
+            - name: GHP_GITHUB_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: ghp-github-app
+                  key: client_secret
+            - name: GHP_ENCRYPTION_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: ghp-app
+                  key: encryption_key
+          volumeMounts:
+            - name: github-app
+              mountPath: /etc/ghp/github-app
+              readOnly: true
+            - name: tls
+              mountPath: /etc/ghp/tls
+              readOnly: true
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /metrics
+              port: metrics
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /metrics
+              port: metrics
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            successThreshold: 1
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: "1"
+              memory: 512Mi
+      volumes:
+        - name: github-app
+          secret:
+            secretName: ghp-github-app
+        - name: tls
+          secret:
+            secretName: ghp-tls
+      restartPolicy: Always
diff --git a/apps/base/ghp/gateway.yaml b/apps/base/ghp/gateway.yaml
new file mode 100644
index 0000000..a61ad4b
--- /dev/null
+++ b/apps/base/ghp/gateway.yaml
@@ -0,0 +1,38 @@
+---
+# Management UI ingress for ghp.k8s.syd1.au.unkin.net via the internal Traefik.
+# TLS is terminated with the ghp-tls Secret produced by the cert-manager
+# Certificate (which also carries the GitHub SANs); no cert-manager annotation
+# here so the two never fight over the same Secret.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: ghp.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: ghp
+  namespace: ghp
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: ghp.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: ghp.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: ghp-tls
+        mode: Terminate
diff --git a/apps/base/ghp/httproute.yaml b/apps/base/ghp/httproute.yaml
new file mode 100644
index 0000000..f692330
--- /dev/null
+++ b/apps/base/ghp/httproute.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: ghp-route
+  namespace: ghp
+spec:
+  hostnames:
+    - ghp.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: ghp
+      sectionName: http
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: ghp
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: ghp
+          port: 80
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
diff --git a/apps/base/ghp/kustomization.yaml b/apps/base/ghp/kustomization.yaml
new file mode 100644
index 0000000..2ba07f8
--- /dev/null
+++ b/apps/base/ghp/kustomization.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - configmap.yaml
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - certificate.yaml
+  - deployment.yaml
+  - migrate-job.yaml
+  - service.yaml
+  - pdb.yaml
+  - vmservicescrape.yaml
+  - gateway.yaml
+  - httproute.yaml
diff --git a/apps/base/ghp/migrate-job.yaml b/apps/base/ghp/migrate-job.yaml
new file mode 100644
index 0000000..6e10b5e
--- /dev/null
+++ b/apps/base/ghp/migrate-job.yaml
@@ -0,0 +1,63 @@
+---
+# Runs the schema migration once per sync, before the Deployment rolls, so the
+# serve replicas never race migrations. Deleted before each re-create so a new
+# image/version re-runs it.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ghp-migrate
+  namespace: ghp
+  annotations:
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 6
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app: ghp-migrate
+    spec:
+      serviceAccountName: ghp
+      automountServiceAccountToken: true
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: migrate
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0
+          imagePullPolicy: IfNotPresent
+          command: ["/ghp", "migrate"]
+          env:
+            - name: GHP_DATABASE_DRIVER
+              value: postgres
+            - name: GHP_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-app
+                  key: username
+            - name: GHP_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-app
+                  key: password
+            - name: GHP_DATABASE_DSN
+              value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require"
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: "1"
+              memory: 512Mi
+            requests:
+              cpu: 250m
+              memory: 256Mi
diff --git a/apps/base/ghp/namespace.yaml b/apps/base/ghp/namespace.yaml
new file mode 100644
index 0000000..ce965f8
--- /dev/null
+++ b/apps/base/ghp/namespace.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ghp
diff --git a/apps/base/ghp/pdb.yaml b/apps/base/ghp/pdb.yaml
new file mode 100644
index 0000000..5953bf0
--- /dev/null
+++ b/apps/base/ghp/pdb.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: ghp
+  namespace: ghp
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: ghp
diff --git a/apps/base/ghp/service.yaml b/apps/base/ghp/service.yaml
new file mode 100644
index 0000000..f1519b9
--- /dev/null
+++ b/apps/base/ghp/service.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ghp
+  namespace: ghp
+  labels:
+    app: ghp
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+    - name: metrics
+      port: 9136
+      protocol: TCP
+      targetPort: metrics
+  selector:
+    app: ghp
+  sessionAffinity: None
+  type: ClusterIP
diff --git a/apps/base/ghp/serviceaccount.yaml b/apps/base/ghp/serviceaccount.yaml
new file mode 100644
index 0000000..cba61fe
--- /dev/null
+++ b/apps/base/ghp/serviceaccount.yaml
@@ -0,0 +1,8 @@
+---
+# Bound to the Vault K8s auth role "ghp" (added in terraform-vault); VSO uses the
+# VaultAuth below to exchange this SA's token for a Vault token and sync kv.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ghp
+  namespace: ghp
diff --git a/apps/base/ghp/vaultauth.yaml b/apps/base/ghp/vaultauth.yaml
new file mode 100644
index 0000000..cfbd3d4
--- /dev/null
+++ b/apps/base/ghp/vaultauth.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: ghp
+spec:
+  allowedNamespaces:
+    - ghp
+  kubernetes:
+    audiences:
+      - vault
+    role: ghp
+    serviceAccount: ghp
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
diff --git a/apps/base/ghp/vaultstaticsecret.yaml b/apps/base/ghp/vaultstaticsecret.yaml
new file mode 100644
index 0000000..c01961b
--- /dev/null
+++ b/apps/base/ghp/vaultstaticsecret.yaml
@@ -0,0 +1,37 @@
+---
+# GitHub App credentials (app_id, client_id, client_secret, private_key).
+# Ben populates kv/kubernetes/ghp/github-app via the runbook; VSO syncs it here.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: ghp-github-app
+  namespace: ghp
+spec:
+  destination:
+    create: true
+    name: ghp-github-app
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/ghp/github-app
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Application encryption key (encryption_key) from kv/kubernetes/ghp/app.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: ghp-app
+  namespace: ghp
+spec:
+  destination:
+    create: true
+    name: ghp-app
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/ghp/app
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
diff --git a/apps/base/ghp/vmservicescrape.yaml b/apps/base/ghp/vmservicescrape.yaml
new file mode 100644
index 0000000..5923764
--- /dev/null
+++ b/apps/base/ghp/vmservicescrape.yaml
@@ -0,0 +1,16 @@
+---
+# Scrape ghp's Prometheus metrics endpoint. Picked up by the observability
+# VMAgent (selectAllByDefault). Estate uses VictoriaMetrics, so VMServiceScrape
+# rather than a prometheus-operator ServiceMonitor.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: ghp
+  namespace: ghp
+spec:
+  selector:
+    matchLabels:
+      app: ghp
+  endpoints:
+    - port: metrics
+      path: /metrics
diff --git a/apps/overlays/au-syd1/ghp/kustomization.yaml b/apps/overlays/au-syd1/ghp/kustomization.yaml
new file mode 100644
index 0000000..80a5cdb
--- /dev/null
+++ b/apps/overlays/au-syd1/ghp/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/ghp
diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml
index 5508107..22f8e08 100644
--- a/argocd/applicationsets/platform.yaml
+++ b/argocd/applicationsets/platform.yaml
@@ -26,6 +26,7 @@ spec:
       - path: apps/overlays/*/elastic-system
       - path: apps/overlays/*/encapi
       - path: apps/overlays/*/externaldns
+      - path: apps/overlays/*/ghp
       - path: apps/overlays/*/gitea
       - path: apps/overlays/*/grafana-system
       - path: apps/overlays/*/inteldeviceplugins-system
diff --git a/argocd/projects/platform.yaml b/argocd/projects/platform.yaml
index e1348a7..9c0ef74 100644
--- a/argocd/projects/platform.yaml
+++ b/argocd/projects/platform.yaml
@@ -39,6 +39,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'externaldns'
       server: https://kubernetes.default.svc
+    - namespace: 'ghp'
+      server: https://kubernetes.default.svc
     - namespace: 'gitea'
       server: https://kubernetes.default.svc
     - namespace: 'jfrog'
-- 
2.47.3


From 42c1794d0ec6ae7875d2690e9d0ef20ee63d66c9 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Thu, 13 Aug 2026 19:53:09 +1000
Subject: [PATCH 2/3] ghp: fix first-install ordering deadlock via sync-waves

The migrate Job was a PreSync hook but connects to postgres-rw, whose CNPG
Cluster + generated postgres-app secret apply in the Sync phase (after all
PreSync hooks). On a fresh install migrate ran before Postgres existed, failed,
exhausted backoffLimit, failed PreSync, and blocked the Sync phase that creates
the DB.

- Move migrate off PreSync to a Sync-phase hook at sync-wave 1.
- Put the CNPG Cluster + backup resources, VSO auth/secrets, and Certificate at
  wave 0 so the DB is Healthy (and creds/cert exist) before migrate runs.
- Put the Deployment + Service/Gateway/HTTPRoute/PDB/VMServiceScrape at wave 2
  so serve starts after migrate completes.
- Add a writable /tmp emptyDir to the serve container and migrate Job (root FS
  is read-only) so codeload/staging writes cannot crash the process.
---
 apps/base/ghp/certificate.yaml       |  2 ++
 apps/base/ghp/cnpg_backup.yaml       |  6 ++++++
 apps/base/ghp/cnpg_cluster.yaml      |  4 ++++
 apps/base/ghp/deployment.yaml        | 11 ++++++++++-
 apps/base/ghp/gateway.yaml           |  1 +
 apps/base/ghp/httproute.yaml         |  2 ++
 apps/base/ghp/migrate-job.yaml       | 17 ++++++++++++++++-
 apps/base/ghp/pdb.yaml               |  2 ++
 apps/base/ghp/service.yaml           |  2 ++
 apps/base/ghp/vaultauth.yaml         |  2 ++
 apps/base/ghp/vaultstaticsecret.yaml |  4 ++++
 apps/base/ghp/vmservicescrape.yaml   |  2 ++
 12 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/apps/base/ghp/certificate.yaml b/apps/base/ghp/certificate.yaml
index 893c396..3e644ae 100644
--- a/apps/base/ghp/certificate.yaml
+++ b/apps/base/ghp/certificate.yaml
@@ -9,6 +9,8 @@ metadata:
   labels:
     app.kubernetes.io/name: ghp
     app.kubernetes.io/instance: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   secretName: ghp-tls
   issuerRef:
diff --git a/apps/base/ghp/cnpg_backup.yaml b/apps/base/ghp/cnpg_backup.yaml
index 323b579..c0df5d6 100644
--- a/apps/base/ghp/cnpg_backup.yaml
+++ b/apps/base/ghp/cnpg_backup.yaml
@@ -6,6 +6,8 @@ kind: ObjectStoreUser
 metadata:
   name: cnpg-ghp-backup
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   displayName: "CNPG backup owner (ghp)"
   uid: cnpg-ghp-backup
@@ -18,6 +20,8 @@ kind: Bucket
 metadata:
   name: cnpg-ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   placementTarget: ec
   bucketName: cnpg-ghp
@@ -35,6 +39,8 @@ kind: ScheduledBackup
 metadata:
   name: cnpg-ghp-nightly
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   schedule: "0 50 1 * * *"
   immediate: false
diff --git a/apps/base/ghp/cnpg_cluster.yaml b/apps/base/ghp/cnpg_cluster.yaml
index bf16319..76a01d0 100644
--- a/apps/base/ghp/cnpg_cluster.yaml
+++ b/apps/base/ghp/cnpg_cluster.yaml
@@ -4,6 +4,10 @@ kind: Cluster
 metadata:
   name: postgres
   namespace: ghp
+  annotations:
+    # Wave 0: DB (and the generated postgres-app Secret) must be Healthy before
+    # the wave-1 migrate Job runs. ArgoCD gates on the Cluster's health status.
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   affinity:
     podAntiAffinityType: preferred
diff --git a/apps/base/ghp/deployment.yaml b/apps/base/ghp/deployment.yaml
index 94787c1..a9204fe 100644
--- a/apps/base/ghp/deployment.yaml
+++ b/apps/base/ghp/deployment.yaml
@@ -5,6 +5,8 @@ metadata:
   name: ghp
   namespace: ghp
   annotations:
+    # Wave 2: serve only after the wave-1 migrate Job completes.
+    argocd.argoproj.io/sync-wave: "2"
     configmap.reloader.stakater.com/auto: "true"
     secret.reloader.stakater.com/reload: "ghp-github-app,ghp-app,ghp-tls,postgres-app"
 spec:
@@ -35,7 +37,7 @@ spec:
           image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0
           imagePullPolicy: IfNotPresent
           # Drop the image's default --migrate so replicas never race migrations;
-          # schema is applied by the PreSync hook Job instead.
+          # schema is applied by the wave-1 migrate hook Job instead.
           command: ["/ghp", "serve"]
           ports:
             - containerPort: 8443
@@ -93,6 +95,8 @@ spec:
             - name: tls
               mountPath: /etc/ghp/tls
               readOnly: true
+            - name: tmp
+              mountPath: /tmp
           livenessProbe:
             failureThreshold: 3
             httpGet:
@@ -133,4 +137,9 @@ spec:
         - name: tls
           secret:
             secretName: ghp-tls
+        # Writable scratch: root FS is read-only. Disk-backed (not memory medium)
+        # so codeload tarball staging doesn't count against the pod memory limit.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 2Gi
       restartPolicy: Always
diff --git a/apps/base/ghp/gateway.yaml b/apps/base/ghp/gateway.yaml
index a61ad4b..80fb011 100644
--- a/apps/base/ghp/gateway.yaml
+++ b/apps/base/ghp/gateway.yaml
@@ -9,6 +9,7 @@ metadata:
   labels:
     traefik.io/instance: internal
   annotations:
+    argocd.argoproj.io/sync-wave: "2"
     external-dns.alpha.kubernetes.io/hostname: ghp.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
   name: ghp
diff --git a/apps/base/ghp/httproute.yaml b/apps/base/ghp/httproute.yaml
index f692330..af65a04 100644
--- a/apps/base/ghp/httproute.yaml
+++ b/apps/base/ghp/httproute.yaml
@@ -4,6 +4,8 @@ kind: HTTPRoute
 metadata:
   name: ghp-route
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   hostnames:
     - ghp.k8s.syd1.au.unkin.net
diff --git a/apps/base/ghp/migrate-job.yaml b/apps/base/ghp/migrate-job.yaml
index 6e10b5e..b067d12 100644
--- a/apps/base/ghp/migrate-job.yaml
+++ b/apps/base/ghp/migrate-job.yaml
@@ -2,14 +2,21 @@
 # Runs the schema migration once per sync, before the Deployment rolls, so the
 # serve replicas never race migrations. Deleted before each re-create so a new
 # image/version re-runs it.
+#
+# Sync-phase hook at wave 1 (NOT PreSync): the CNPG Cluster + generated
+# postgres-app Secret apply at wave 0 and ArgoCD waits for the Cluster to be
+# Healthy before starting wave 1, so on a fresh install Postgres exists before
+# migrate connects. (A PreSync hook would run before the Sync phase that creates
+# the DB, deadlocking the first install.)
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: ghp-migrate
   namespace: ghp
   annotations:
-    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    argocd.argoproj.io/sync-wave: "1"
 spec:
   backoffLimit: 6
   ttlSecondsAfterFinished: 600
@@ -48,6 +55,9 @@ spec:
                   key: password
             - name: GHP_DATABASE_DSN
               value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require"
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -61,3 +71,8 @@ spec:
             requests:
               cpu: 250m
               memory: 256Mi
+      volumes:
+        # Writable scratch: root FS is read-only, so give the migrator a /tmp.
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi
diff --git a/apps/base/ghp/pdb.yaml b/apps/base/ghp/pdb.yaml
index 5953bf0..4a47f6d 100644
--- a/apps/base/ghp/pdb.yaml
+++ b/apps/base/ghp/pdb.yaml
@@ -4,6 +4,8 @@ kind: PodDisruptionBudget
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   minAvailable: 1
   selector:
diff --git a/apps/base/ghp/service.yaml b/apps/base/ghp/service.yaml
index f1519b9..7c3e1d3 100644
--- a/apps/base/ghp/service.yaml
+++ b/apps/base/ghp/service.yaml
@@ -6,6 +6,8 @@ metadata:
   namespace: ghp
   labels:
     app: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   internalTrafficPolicy: Cluster
   ports:
diff --git a/apps/base/ghp/vaultauth.yaml b/apps/base/ghp/vaultauth.yaml
index cfbd3d4..1678ba0 100644
--- a/apps/base/ghp/vaultauth.yaml
+++ b/apps/base/ghp/vaultauth.yaml
@@ -4,6 +4,8 @@ kind: VaultAuth
 metadata:
   name: default
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   allowedNamespaces:
     - ghp
diff --git a/apps/base/ghp/vaultstaticsecret.yaml b/apps/base/ghp/vaultstaticsecret.yaml
index c01961b..a44e926 100644
--- a/apps/base/ghp/vaultstaticsecret.yaml
+++ b/apps/base/ghp/vaultstaticsecret.yaml
@@ -6,6 +6,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-github-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true
@@ -24,6 +26,8 @@ kind: VaultStaticSecret
 metadata:
   name: ghp-app
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
 spec:
   destination:
     create: true
diff --git a/apps/base/ghp/vmservicescrape.yaml b/apps/base/ghp/vmservicescrape.yaml
index 5923764..a91bde9 100644
--- a/apps/base/ghp/vmservicescrape.yaml
+++ b/apps/base/ghp/vmservicescrape.yaml
@@ -7,6 +7,8 @@ kind: VMServiceScrape
 metadata:
   name: ghp
   namespace: ghp
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
 spec:
   selector:
     matchLabels:
-- 
2.47.3


From 5faaff8d19e2fe582ab9ea6be797c802b68036db Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Thu, 13 Aug 2026 20:01:43 +1000
Subject: [PATCH 3/3] ghp: use the estate templated default Vault convention

Drop the bespoke ghp SA/role/policy (terraform-vault#120 closed). The default
k8s auth role, bound to SA default in every namespace, already has a templated
read grant on kv/kubernetes/namespace/<ns>/default/*, so ghp needs zero
terraform-vault change.

- Remove the custom ServiceAccount; run as the namespace default SA.
- Deployment + migrate Job: serviceAccountName default.
- VaultAuth: role/serviceAccount default (mirrors artifactapi).
- VaultStaticSecrets: source paths move to the templated location
  kubernetes/namespace/ghp/default/{github-app,app}; Secret names unchanged.
---
 apps/base/ghp/deployment.yaml        |  2 +-
 apps/base/ghp/kustomization.yaml     |  1 -
 apps/base/ghp/migrate-job.yaml       |  2 +-
 apps/base/ghp/serviceaccount.yaml    |  8 --------
 apps/base/ghp/vaultauth.yaml         |  4 ++--
 apps/base/ghp/vaultstaticsecret.yaml | 13 ++++++++-----
 6 files changed, 12 insertions(+), 18 deletions(-)
 delete mode 100644 apps/base/ghp/serviceaccount.yaml

diff --git a/apps/base/ghp/deployment.yaml b/apps/base/ghp/deployment.yaml
index a9204fe..ab4013b 100644
--- a/apps/base/ghp/deployment.yaml
+++ b/apps/base/ghp/deployment.yaml
@@ -23,7 +23,7 @@ spec:
       labels:
         app: ghp
     spec:
-      serviceAccountName: ghp
+      serviceAccountName: default
       automountServiceAccountToken: true
       securityContext:
         runAsNonRoot: true
diff --git a/apps/base/ghp/kustomization.yaml b/apps/base/ghp/kustomization.yaml
index 2ba07f8..a4db15f 100644
--- a/apps/base/ghp/kustomization.yaml
+++ b/apps/base/ghp/kustomization.yaml
@@ -4,7 +4,6 @@ kind: Kustomization
 
 resources:
   - namespace.yaml
-  - serviceaccount.yaml
   - configmap.yaml
   - cnpg_cluster.yaml
   - cnpg_backup.yaml
diff --git a/apps/base/ghp/migrate-job.yaml b/apps/base/ghp/migrate-job.yaml
index b067d12..443feef 100644
--- a/apps/base/ghp/migrate-job.yaml
+++ b/apps/base/ghp/migrate-job.yaml
@@ -25,7 +25,7 @@ spec:
       labels:
         app: ghp-migrate
     spec:
-      serviceAccountName: ghp
+      serviceAccountName: default
       automountServiceAccountToken: true
       restartPolicy: Never
       securityContext:
diff --git a/apps/base/ghp/serviceaccount.yaml b/apps/base/ghp/serviceaccount.yaml
deleted file mode 100644
index cba61fe..0000000
--- a/apps/base/ghp/serviceaccount.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-# Bound to the Vault K8s auth role "ghp" (added in terraform-vault); VSO uses the
-# VaultAuth below to exchange this SA's token for a Vault token and sync kv.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: ghp
-  namespace: ghp
diff --git a/apps/base/ghp/vaultauth.yaml b/apps/base/ghp/vaultauth.yaml
index 1678ba0..05a1955 100644
--- a/apps/base/ghp/vaultauth.yaml
+++ b/apps/base/ghp/vaultauth.yaml
@@ -12,8 +12,8 @@ spec:
   kubernetes:
     audiences:
       - vault
-    role: ghp
-    serviceAccount: ghp
+    role: default
+    serviceAccount: default
     tokenExpirationSeconds: 600
   method: kubernetes
   mount: k8s/au/syd1
diff --git a/apps/base/ghp/vaultstaticsecret.yaml b/apps/base/ghp/vaultstaticsecret.yaml
index a44e926..1c454f3 100644
--- a/apps/base/ghp/vaultstaticsecret.yaml
+++ b/apps/base/ghp/vaultstaticsecret.yaml
@@ -1,6 +1,8 @@
 ---
-# GitHub App credentials (app_id, client_id, client_secret, private_key).
-# Ben populates kv/kubernetes/ghp/github-app via the runbook; VSO syncs it here.
+# GitHub App credentials (app_id, client_id, client_secret, private_key). Ben
+# populates kv/kubernetes/namespace/ghp/default/github-app via the runbook; the
+# default k8s role's templated policy already grants read here, so no
+# terraform-vault change is needed. VSO syncs it into the ghp-github-app Secret.
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
@@ -15,12 +17,13 @@ spec:
     overwrite: true
   hmacSecretData: true
   mount: kv
-  path: kubernetes/ghp/github-app
+  path: kubernetes/namespace/ghp/default/github-app
   refreshAfter: 5m
   type: kv-v2
   vaultAuthRef: default
 ---
-# Application encryption key (encryption_key) from kv/kubernetes/ghp/app.
+# Application encryption key (encryption_key) from
+# kv/kubernetes/namespace/ghp/default/app.
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
@@ -35,7 +38,7 @@ spec:
     overwrite: true
   hmacSecretData: true
   mount: kv
-  path: kubernetes/ghp/app
+  path: kubernetes/namespace/ghp/default/app
   refreshAfter: 5m
   type: kv-v2
   vaultAuthRef: default
-- 
2.47.3