From e5573cf6fdf147f7b46ecdbd2840c4bbe4922928 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 27 Sep 2026 11:08:56 +1000 Subject: [PATCH] Drop the gocache serve Deployment and nginx sidecar `connect` mode gives the client no local cache, so every cache hit is a network round trip -- about 2.7x slower than pointing the plugin straight at S3. - Remove the go-cache-plugin serve Deployment, its nginx stream-proxy ConfigMap, and the PureLB Service on 198.18.200.11 - Keep the gocache bucket and gocache-s3 Secret: clients now run the plugin in direct mode with those credentials --- .../woodpecker/configmap_gocache-nginx.yaml | 39 ------ apps/base/woodpecker/deployment_gocache.yaml | 131 ------------------ apps/base/woodpecker/kustomization.yaml | 3 - apps/base/woodpecker/service_gocache.yaml | 23 --- 4 files changed, 196 deletions(-) delete mode 100644 apps/base/woodpecker/configmap_gocache-nginx.yaml delete mode 100644 apps/base/woodpecker/deployment_gocache.yaml delete mode 100644 apps/base/woodpecker/service_gocache.yaml diff --git a/apps/base/woodpecker/configmap_gocache-nginx.yaml b/apps/base/woodpecker/configmap_gocache-nginx.yaml deleted file mode 100644 index ca1eaeb..0000000 --- a/apps/base/woodpecker/configmap_gocache-nginx.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: gocache-nginx - namespace: woodpecker - labels: - app.kubernetes.io/name: gocache - app.kubernetes.io/component: proxy -data: - nginx.conf: | - worker_processes auto; - error_log /dev/stderr warn; - pid /tmp/nginx.pid; - - events { - worker_connections 512; - } - - # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}. - stream { - server { - listen 9090; - - # The protocol has no authentication: anyone who can reach this port can - # write cache entries, which become code in every build that reads them. - # Loopback is the kubectl port-forward fallback; in a pod netns it is - # only these two containers. - allow 127.0.0.1/32; - allow 10.10.12.200/32; - allow 10.42.0.0/16; - deny all; - - # A connect session lasts the whole build; the 10m default cuts long builds off. - proxy_timeout 2h; - proxy_connect_timeout 5s; - proxy_pass 127.0.0.1:9080; - } - } diff --git a/apps/base/woodpecker/deployment_gocache.yaml b/apps/base/woodpecker/deployment_gocache.yaml deleted file mode 100644 index dad3f9d..0000000 --- a/apps/base/woodpecker/deployment_gocache.yaml +++ /dev/null @@ -1,131 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gocache - namespace: woodpecker - annotations: - configmap.reloader.stakater.com/reload: "gocache-nginx" - secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert" - labels: - app.kubernetes.io/name: gocache - app.kubernetes.io/component: cache -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: gocache - template: - metadata: - labels: - app.kubernetes.io/name: gocache - app.kubernetes.io/component: cache - spec: - serviceAccountName: default - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - fsGroup: 65532 - seccompProfile: - type: RuntimeDefault - containers: - - name: go-cache-plugin - image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0 - imagePullPolicy: IfNotPresent - # Root flags must precede the subcommand; only --plugin belongs to serve. - args: - - --cache-dir=/var/cache/gocache - - --bucket=gocache - # Explicit region skips the GetBucketLocation probe, which RGW handles poorly. - - --region=us-east-1 - - --s3-endpoint-url=https://s3.ceph.unkin.net - - --s3-path-style - - serve - - --plugin=9080 - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: gocache-s3 - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: gocache-s3 - key: AWS_SECRET_ACCESS_KEY - # s3.ceph.unkin.net is served by the estate CA, not a public root. - - name: AWS_CA_BUNDLE - value: /etc/ssl/vault-ca/ca.crt - volumeMounts: - - name: cache - mountPath: /var/cache/gocache - - name: vault-ca - mountPath: /etc/ssl/vault-ca - readOnly: true - securityContext: - runAsUser: 65532 - runAsGroup: 65532 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: "2" - memory: 2Gi - - name: nginx - image: docker.io/nginx:1.29.8-alpine - imagePullPolicy: IfNotPresent - # Bypass the image entrypoint: its config scripts write to a read-only rootfs. - command: - - nginx - - -g - - daemon off; - ports: - - containerPort: 9090 - name: gocache - protocol: TCP - volumeMounts: - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - - name: tmp - mountPath: /tmp - securityContext: - runAsUser: 101 - runAsGroup: 101 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 500m - memory: 128Mi - volumes: - # Staging cache in front of S3: losing it costs a repopulate, not data. - - name: cache - emptyDir: - sizeLimit: 20Gi - - name: nginx-config - configMap: - name: gocache-nginx - - name: tmp - emptyDir: {} - - name: vault-ca - secret: - secretName: vault-ca-cert - items: - - key: ca.crt - path: ca.crt diff --git a/apps/base/woodpecker/kustomization.yaml b/apps/base/woodpecker/kustomization.yaml index ecddb7d..ae394e2 100644 --- a/apps/base/woodpecker/kustomization.yaml +++ b/apps/base/woodpecker/kustomization.yaml @@ -8,9 +8,6 @@ resources: - cnpg_backup.yaml - cnpg_pooler.yaml - gocache_bucket.yaml - - configmap_gocache-nginx.yaml - - deployment_gocache.yaml - - service_gocache.yaml - serviceaccount_arrproxy_ci.yaml - serviceaccount_autobackup_operator_ci.yaml - serviceaccount_ghp.yaml diff --git a/apps/base/woodpecker/service_gocache.yaml b/apps/base/woodpecker/service_gocache.yaml deleted file mode 100644 index 7cf0d7e..0000000 --- a/apps/base/woodpecker/service_gocache.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: gocache - namespace: woodpecker - annotations: - purelb.io/addresses: 198.18.200.11 - purelb.io/service-group: common - labels: - app.kubernetes.io/name: gocache -spec: - type: LoadBalancer - # Cluster SNATs off-node traffic to a node address, which would defeat the - # nginx allow rules; Local preserves the wireguard client IP. - externalTrafficPolicy: Local - selector: - app.kubernetes.io/name: gocache - ports: - - name: gocache - port: 9090 - targetPort: gocache - protocol: TCP