From 693f541840fa637f74cb17df992245399305efa8 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 27 Sep 2026 00:25:04 +1000 Subject: [PATCH] Add gocache serve Deployment for the shared Go build cache go-cache-plugin serve binds 127.0.0.1 only, so a Service cannot reach it directly and CI/developer builds have no way to use the S3-backed cache. - Run go-cache-plugin serve against the gocache RGW bucket, path-style, with an explicit region to skip the GetBucketLocation probe - Add an nginx sidecar stream-proxying 9090 to the loopback plugin port - Restrict the listener to the workstation and pod CIDRs: GOCACHEPROG is unauthenticated and a poisoned entry runs in every consuming build - Stage the cache on an emptyDir; loss costs a repopulate from S3 --- .../woodpecker/configmap_gocache-nginx.yaml | 37 +++++ apps/base/woodpecker/deployment_gocache.yaml | 131 ++++++++++++++++++ apps/base/woodpecker/kustomization.yaml | 3 + apps/base/woodpecker/service_gocache.yaml | 17 +++ 4 files changed, 188 insertions(+) create mode 100644 apps/base/woodpecker/configmap_gocache-nginx.yaml create mode 100644 apps/base/woodpecker/deployment_gocache.yaml create mode 100644 apps/base/woodpecker/service_gocache.yaml diff --git a/apps/base/woodpecker/configmap_gocache-nginx.yaml b/apps/base/woodpecker/configmap_gocache-nginx.yaml new file mode 100644 index 0000000..2c7fe91 --- /dev/null +++ b/apps/base/woodpecker/configmap_gocache-nginx.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gocache-nginx + namespace: woodpecker + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: proxy +data: + nginx.conf: | + worker_processes auto; + error_log /dev/stderr warn; + pid /tmp/nginx.pid; + + events { + worker_connections 512; + } + + # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}. + stream { + server { + listen 9090; + + # The protocol has no authentication: anyone who can reach this port can + # write cache entries, which become code in every build that reads them. + allow 127.0.0.1/32; + allow 10.10.12.200/32; + allow 10.42.0.0/16; + deny all; + + # A connect session lasts the whole build; the 10m default cuts long builds off. + proxy_timeout 2h; + proxy_connect_timeout 5s; + proxy_pass 127.0.0.1:9080; + } + } diff --git a/apps/base/woodpecker/deployment_gocache.yaml b/apps/base/woodpecker/deployment_gocache.yaml new file mode 100644 index 0000000..dad3f9d --- /dev/null +++ b/apps/base/woodpecker/deployment_gocache.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gocache + namespace: woodpecker + annotations: + configmap.reloader.stakater.com/reload: "gocache-nginx" + secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert" + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: cache +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: gocache + template: + metadata: + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: cache + spec: + serviceAccountName: default + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: go-cache-plugin + image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0 + imagePullPolicy: IfNotPresent + # Root flags must precede the subcommand; only --plugin belongs to serve. + args: + - --cache-dir=/var/cache/gocache + - --bucket=gocache + # Explicit region skips the GetBucketLocation probe, which RGW handles poorly. + - --region=us-east-1 + - --s3-endpoint-url=https://s3.ceph.unkin.net + - --s3-path-style + - serve + - --plugin=9080 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: gocache-s3 + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: gocache-s3 + key: AWS_SECRET_ACCESS_KEY + # s3.ceph.unkin.net is served by the estate CA, not a public root. + - name: AWS_CA_BUNDLE + value: /etc/ssl/vault-ca/ca.crt + volumeMounts: + - name: cache + mountPath: /var/cache/gocache + - name: vault-ca + mountPath: /etc/ssl/vault-ca + readOnly: true + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi + - name: nginx + image: docker.io/nginx:1.29.8-alpine + imagePullPolicy: IfNotPresent + # Bypass the image entrypoint: its config scripts write to a read-only rootfs. + command: + - nginx + - -g + - daemon off; + ports: + - containerPort: 9090 + name: gocache + protocol: TCP + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: tmp + mountPath: /tmp + securityContext: + runAsUser: 101 + runAsGroup: 101 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 500m + memory: 128Mi + volumes: + # Staging cache in front of S3: losing it costs a repopulate, not data. + - name: cache + emptyDir: + sizeLimit: 20Gi + - name: nginx-config + configMap: + name: gocache-nginx + - name: tmp + emptyDir: {} + - name: vault-ca + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt diff --git a/apps/base/woodpecker/kustomization.yaml b/apps/base/woodpecker/kustomization.yaml index ae394e2..ecddb7d 100644 --- a/apps/base/woodpecker/kustomization.yaml +++ b/apps/base/woodpecker/kustomization.yaml @@ -8,6 +8,9 @@ resources: - cnpg_backup.yaml - cnpg_pooler.yaml - gocache_bucket.yaml + - configmap_gocache-nginx.yaml + - deployment_gocache.yaml + - service_gocache.yaml - serviceaccount_arrproxy_ci.yaml - serviceaccount_autobackup_operator_ci.yaml - serviceaccount_ghp.yaml diff --git a/apps/base/woodpecker/service_gocache.yaml b/apps/base/woodpecker/service_gocache.yaml new file mode 100644 index 0000000..cd51221 --- /dev/null +++ b/apps/base/woodpecker/service_gocache.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: gocache + namespace: woodpecker + labels: + app.kubernetes.io/name: gocache +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gocache + ports: + - name: gocache + port: 9090 + targetPort: gocache + protocol: TCP