diff --git a/apps/base/ghp/certificate.yaml b/apps/base/ghp/certificate.yaml new file mode 100644 index 0000000..3e644ae --- /dev/null +++ b/apps/base/ghp/certificate.yaml @@ -0,0 +1,28 @@ +--- +# Serving cert off the internal Vault-PKI CA (agents already trust vault-ca-cert). +# SANs cover the GitHub endpoints ghp impersonates plus its own management host. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ghp-tls + namespace: ghp + labels: + app.kubernetes.io/name: ghp + app.kubernetes.io/instance: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + secretName: ghp-tls + issuerRef: + kind: ClusterIssuer + name: vault-issuer + commonName: ghp.k8s.syd1.au.unkin.net + dnsNames: + - github.com + - api.github.com + - codeload.github.com + - "*.githubcopilot.com" + - ghp.k8s.syd1.au.unkin.net + privateKey: + algorithm: RSA + size: 4096 diff --git a/apps/base/ghp/cnpg_backup.yaml b/apps/base/ghp/cnpg_backup.yaml new file mode 100644 index 0000000..c0df5d6 --- /dev/null +++ b/apps/base/ghp/cnpg_backup.yaml @@ -0,0 +1,50 @@ +--- +# Ceph RGW (S3) backup target for the ghp CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-ghp-backup + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + displayName: "CNPG backup owner (ghp)" + uid: cnpg-ghp-backup + maxBuckets: 5 + secretName: cnpg-ghp-backup-s3 + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-ghp + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + placementTarget: ec + bucketName: cnpg-ghp + ownerRef: cnpg-ghp-backup + versioning: false + tags: + app: ghp + purpose: cnpg-backup + retainOnDelete: true +--- +# Nightly base backup; continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore. Staggered off other clusters' schedules. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-ghp-nightly + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + schedule: "0 50 1 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: postgres diff --git a/apps/base/ghp/cnpg_cluster.yaml b/apps/base/ghp/cnpg_cluster.yaml new file mode 100644 index 0000000..76a01d0 --- /dev/null +++ b/apps/base/ghp/cnpg_cluster.yaml @@ -0,0 +1,117 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: postgres + namespace: ghp + annotations: + # Wave 0: DB (and the generated postgres-app Secret) must be Healthy before + # the wave-1 migrate Job runs. ArgoCD gates on the Cluster's health status. + argocd.argoproj.io/sync-wave: "0" +spec: + affinity: + podAntiAffinityType: preferred + backup: + retentionPolicy: 30d + barmanObjectStore: + destinationPath: s3://cnpg-ghp + endpointURL: https://s3.ceph.unkin.net + endpointCA: + name: vault-ca-cert + key: ca.crt + s3Credentials: + accessKeyId: + name: cnpg-ghp-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-ghp-backup-s3 + key: AWS_SECRET_ACCESS_KEY + serverName: ghp + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 + bootstrap: + initdb: + # No secret ref: CNPG mints the owner credentials and publishes them in the + # generated "postgres-app" Secret, which the Deployment reads to build the DSN. + database: ghp + encoding: UTF8 + localeCType: C + localeCollate: C + owner: ghp + enablePDB: true + enableSuperuserAccess: false + failoverDelay: 0 + imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie + instances: 3 + logLevel: info + maxSyncReplicas: 0 + minSyncReplicas: 0 + monitoring: + customQueriesConfigMap: + - key: queries + name: cnpg-default-monitoring + disableDefaultQueries: false + enablePodMonitor: false + postgresql: + parameters: + archive_mode: "on" + archive_timeout: 5min + dynamic_shared_memory_type: posix + effective_cache_size: 256MB + full_page_writes: "on" + log_destination: csvlog + log_directory: /controller/log + log_filename: postgres + log_rotation_age: "0" + log_rotation_size: "0" + log_truncate_on_rotation: "false" + logging_collector: "on" + max_connections: "200" + max_parallel_workers: "16" + max_replication_slots: "16" + max_worker_processes: "16" + shared_buffers: 128MB + shared_memory_type: mmap + ssl_max_protocol_version: TLSv1.3 + ssl_min_protocol_version: TLSv1.3 + wal_keep_size: 256MB + wal_level: logical + wal_log_hints: "on" + wal_receiver_timeout: 5s + wal_sender_timeout: 5s + syncReplicaElectionConstraint: + enabled: false + primaryUpdateMethod: restart + primaryUpdateStrategy: unsupervised + probes: + liveness: + isolationCheck: + connectionTimeout: 1000 + enabled: true + requestTimeout: 1000 + replicationSlots: + highAvailability: + enabled: true + slotPrefix: _cnpg_ + synchronizeReplicas: + enabled: true + updateInterval: 30 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 256Mi + smartShutdownTimeout: 180 + startDelay: 3600 + stopDelay: 1800 + storage: + resizeInUseVolumes: true + size: 20Gi + storageClass: cephrbd-fast-delete + switchoverDelay: 3600 diff --git a/apps/base/ghp/configmap.yaml b/apps/base/ghp/configmap.yaml new file mode 100644 index 0000000..f35f4ed --- /dev/null +++ b/apps/base/ghp/configmap.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ghp-env + namespace: ghp +data: + GHP_DATABASE_DRIVER: postgres + # nonroot cannot bind <1024; listen high and remap in the Service. + GHP_SERVER_HTTPS_LISTEN: ":8443" + GHP_SERVER_HTTP_LISTEN: ":8080" + GHP_METRICS_LISTEN: ":9136" + GHP_METRICS_ENABLED: "true" + GHP_SERVER_BASE_URL: https://ghp.k8s.syd1.au.unkin.net + GHP_SERVER_MANAGEMENT_HOST: ghp.k8s.syd1.au.unkin.net + # private_key key from the ghp-github-app Secret, mounted as a file. + GHP_GITHUB_PRIVATE_KEY_FILE: /etc/ghp/github-app/private_key + # cert-manager Certificate ghp-tls, mounted from the ghp-tls Secret. + GHP_TLS_CERT_FILE: /etc/ghp/tls/tls.crt + GHP_TLS_KEY_FILE: /etc/ghp/tls/tls.key + # PLACEHOLDER: set to Ben's GitHub username before ghp will admit an admin. + GHP_ADMINS: "REPLACE_ME_ben_github_username" diff --git a/apps/base/ghp/deployment.yaml b/apps/base/ghp/deployment.yaml new file mode 100644 index 0000000..ab4013b --- /dev/null +++ b/apps/base/ghp/deployment.yaml @@ -0,0 +1,145 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ghp + namespace: ghp + annotations: + # Wave 2: serve only after the wave-1 migrate Job completes. + argocd.argoproj.io/sync-wave: "2" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "ghp-github-app,ghp-app,ghp-tls,postgres-app" +spec: + replicas: 2 + selector: + matchLabels: + app: ghp + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: ghp + spec: + serviceAccountName: default + automountServiceAccountToken: true + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: ghp + image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0 + imagePullPolicy: IfNotPresent + # Drop the image's default --migrate so replicas never race migrations; + # schema is applied by the wave-1 migrate hook Job instead. + command: ["/ghp", "serve"] + ports: + - containerPort: 8443 + name: https + protocol: TCP + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 9136 + name: metrics + protocol: TCP + envFrom: + - configMapRef: + name: ghp-env + optional: false + env: + # DSN assembled from the CNPG-generated postgres-app Secret; $(VAR) + # expansion resolves the two env entries defined above it. + - name: GHP_DB_USER + valueFrom: + secretKeyRef: + name: postgres-app + key: username + - name: GHP_DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-app + key: password + - name: GHP_DATABASE_DSN + value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require" + - name: GHP_GITHUB_APP_ID + valueFrom: + secretKeyRef: + name: ghp-github-app + key: app_id + - name: GHP_GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: ghp-github-app + key: client_id + - name: GHP_GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: ghp-github-app + key: client_secret + - name: GHP_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: ghp-app + key: encryption_key + volumeMounts: + - name: github-app + mountPath: /etc/ghp/github-app + readOnly: true + - name: tls + mountPath: /etc/ghp/tls + readOnly: true + - name: tmp + mountPath: /tmp + livenessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: metrics + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: metrics + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 512Mi + volumes: + - name: github-app + secret: + secretName: ghp-github-app + - name: tls + secret: + secretName: ghp-tls + # Writable scratch: root FS is read-only. Disk-backed (not memory medium) + # so codeload tarball staging doesn't count against the pod memory limit. + - name: tmp + emptyDir: + sizeLimit: 2Gi + restartPolicy: Always diff --git a/apps/base/ghp/gateway.yaml b/apps/base/ghp/gateway.yaml new file mode 100644 index 0000000..80fb011 --- /dev/null +++ b/apps/base/ghp/gateway.yaml @@ -0,0 +1,39 @@ +--- +# Management UI ingress for ghp.k8s.syd1.au.unkin.net via the internal Traefik. +# TLS is terminated with the ghp-tls Secret produced by the cert-manager +# Certificate (which also carries the GitHub SANs); no cert-manager annotation +# here so the two never fight over the same Secret. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + labels: + traefik.io/instance: internal + annotations: + argocd.argoproj.io/sync-wave: "2" + external-dns.alpha.kubernetes.io/hostname: ghp.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.4 + name: ghp + namespace: ghp +spec: + gatewayClassName: traefik-internal + listeners: + - allowedRoutes: + namespaces: + from: Same + hostname: ghp.k8s.syd1.au.unkin.net + name: http + port: 80 + protocol: HTTP + - allowedRoutes: + namespaces: + from: Same + hostname: ghp.k8s.syd1.au.unkin.net + name: https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - group: "" + kind: Secret + name: ghp-tls + mode: Terminate diff --git a/apps/base/ghp/httproute.yaml b/apps/base/ghp/httproute.yaml new file mode 100644 index 0000000..af65a04 --- /dev/null +++ b/apps/base/ghp/httproute.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: ghp-route + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + hostnames: + - ghp.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: ghp + sectionName: http + - group: gateway.networking.k8s.io + kind: Gateway + name: ghp + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: ghp + port: 80 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/ghp/kustomization.yaml b/apps/base/ghp/kustomization.yaml new file mode 100644 index 0000000..a4db15f --- /dev/null +++ b/apps/base/ghp/kustomization.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap.yaml + - cnpg_cluster.yaml + - cnpg_backup.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - certificate.yaml + - deployment.yaml + - migrate-job.yaml + - service.yaml + - pdb.yaml + - vmservicescrape.yaml + - gateway.yaml + - httproute.yaml diff --git a/apps/base/ghp/migrate-job.yaml b/apps/base/ghp/migrate-job.yaml new file mode 100644 index 0000000..443feef --- /dev/null +++ b/apps/base/ghp/migrate-job.yaml @@ -0,0 +1,78 @@ +--- +# Runs the schema migration once per sync, before the Deployment rolls, so the +# serve replicas never race migrations. Deleted before each re-create so a new +# image/version re-runs it. +# +# Sync-phase hook at wave 1 (NOT PreSync): the CNPG Cluster + generated +# postgres-app Secret apply at wave 0 and ArgoCD waits for the Cluster to be +# Healthy before starting wave 1, so on a fresh install Postgres exists before +# migrate connects. (A PreSync hook would run before the Sync phase that creates +# the DB, deadlocking the first install.) +apiVersion: batch/v1 +kind: Job +metadata: + name: ghp-migrate + namespace: ghp + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + argocd.argoproj.io/sync-wave: "1" +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app: ghp-migrate + spec: + serviceAccountName: default + automountServiceAccountToken: true + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: migrate + image: artifactapi.k8s.syd1.au.unkin.net/ghcr/goodtune/ghp:0.20.0 + imagePullPolicy: IfNotPresent + command: ["/ghp", "migrate"] + env: + - name: GHP_DATABASE_DRIVER + value: postgres + - name: GHP_DB_USER + valueFrom: + secretKeyRef: + name: postgres-app + key: username + - name: GHP_DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-app + key: password + - name: GHP_DATABASE_DSN + value: "postgres://$(GHP_DB_USER):$(GHP_DB_PASSWORD)@postgres-rw.ghp.svc:5432/ghp?sslmode=require" + volumeMounts: + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: "1" + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi + volumes: + # Writable scratch: root FS is read-only, so give the migrator a /tmp. + - name: tmp + emptyDir: + sizeLimit: 256Mi diff --git a/apps/base/ghp/namespace.yaml b/apps/base/ghp/namespace.yaml new file mode 100644 index 0000000..ce965f8 --- /dev/null +++ b/apps/base/ghp/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ghp diff --git a/apps/base/ghp/pdb.yaml b/apps/base/ghp/pdb.yaml new file mode 100644 index 0000000..4a47f6d --- /dev/null +++ b/apps/base/ghp/pdb.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ghp + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + minAvailable: 1 + selector: + matchLabels: + app: ghp diff --git a/apps/base/ghp/service.yaml b/apps/base/ghp/service.yaml new file mode 100644 index 0000000..7c3e1d3 --- /dev/null +++ b/apps/base/ghp/service.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: ghp + namespace: ghp + labels: + app: ghp + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + internalTrafficPolicy: Cluster + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: metrics + port: 9136 + protocol: TCP + targetPort: metrics + selector: + app: ghp + sessionAffinity: None + type: ClusterIP diff --git a/apps/base/ghp/vaultauth.yaml b/apps/base/ghp/vaultauth.yaml new file mode 100644 index 0000000..05a1955 --- /dev/null +++ b/apps/base/ghp/vaultauth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + allowedNamespaces: + - ghp + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/ghp/vaultstaticsecret.yaml b/apps/base/ghp/vaultstaticsecret.yaml new file mode 100644 index 0000000..1c454f3 --- /dev/null +++ b/apps/base/ghp/vaultstaticsecret.yaml @@ -0,0 +1,44 @@ +--- +# GitHub App credentials (app_id, client_id, client_secret, private_key). Ben +# populates kv/kubernetes/namespace/ghp/default/github-app via the runbook; the +# default k8s role's templated policy already grants read here, so no +# terraform-vault change is needed. VSO syncs it into the ghp-github-app Secret. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: ghp-github-app + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + destination: + create: true + name: ghp-github-app + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/ghp/default/github-app + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default +--- +# Application encryption key (encryption_key) from +# kv/kubernetes/namespace/ghp/default/app. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: ghp-app + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + destination: + create: true + name: ghp-app + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/ghp/default/app + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/ghp/vmservicescrape.yaml b/apps/base/ghp/vmservicescrape.yaml new file mode 100644 index 0000000..a91bde9 --- /dev/null +++ b/apps/base/ghp/vmservicescrape.yaml @@ -0,0 +1,18 @@ +--- +# Scrape ghp's Prometheus metrics endpoint. Picked up by the observability +# VMAgent (selectAllByDefault). Estate uses VictoriaMetrics, so VMServiceScrape +# rather than a prometheus-operator ServiceMonitor. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: ghp + namespace: ghp + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + selector: + matchLabels: + app: ghp + endpoints: + - port: metrics + path: /metrics diff --git a/apps/overlays/au-syd1/ghp/kustomization.yaml b/apps/overlays/au-syd1/ghp/kustomization.yaml new file mode 100644 index 0000000..80a5cdb --- /dev/null +++ b/apps/overlays/au-syd1/ghp/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/ghp diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index 5508107..22f8e08 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -26,6 +26,7 @@ spec: - path: apps/overlays/*/elastic-system - path: apps/overlays/*/encapi - path: apps/overlays/*/externaldns + - path: apps/overlays/*/ghp - path: apps/overlays/*/gitea - path: apps/overlays/*/grafana-system - path: apps/overlays/*/inteldeviceplugins-system diff --git a/argocd/projects/platform.yaml b/argocd/projects/platform.yaml index e1348a7..9c0ef74 100644 --- a/argocd/projects/platform.yaml +++ b/argocd/projects/platform.yaml @@ -39,6 +39,8 @@ spec: server: https://kubernetes.default.svc - namespace: 'externaldns' server: https://kubernetes.default.svc + - namespace: 'ghp' + server: https://kubernetes.default.svc - namespace: 'gitea' server: https://kubernetes.default.svc - namespace: 'jfrog'