Drop the gocache serve Deployment and nginx sidecar (#499)

`connect` mode gives the client no local cache, so every cache operation is a network round trip -- about 2.7x slower than pointing the plugin straight at S3. Developers and CI run the plugin in direct mode instead.

- Remove the `go-cache-plugin serve` Deployment and its nginx stream-proxy ConfigMap
- Remove the PureLB Service, freeing `198.18.200.11`
- Keep the `gocache` bucket and `gocache-s3` Secret for direct-mode clients

Reviewed-on: #499
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/bind-external/acme-unkin-net/a/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ns1.yaml

apps/base/bind-external/acme-unkin-net/a/ns1.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-ns1-a
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  name: ns1
-  type: A
-  ttl: 3600
-  values:
-    # Public address of this cluster's external BIND, same target as
-    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
-    # NS name must still reach the zone.
-    - 103.216.191.185

apps/base/bind-external/acme-unkin-net/kustomization.yaml

@@ -1,11 +0,0 @@
----
-# Authoritative delegation records for acme.unkin.net. Without these the zone
-# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
-# primary pod IP), which is unroutable off-cluster and goes stale on
-# reschedule. DNSRecords must live in the same namespace as their BindZone.
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ns
-  - a

apps/base/bind-external/acme-unkin-net/ns/apex.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-apex-ns
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  # "@" is the zone apex.
-  name: "@"
-  type: NS
-  ttl: 3600
-  values:
-    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
-    # child needs no glue of its own.
-    - acme-ns1.unkin.net.

apps/base/bind-external/acme-unkin-net/ns/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - apex.yaml

apps/base/bind-external/kustomization.yaml

@@ -7,5 +7,5 @@ resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
-  - acme-unkin-net
+  - records.yaml
   - agent-dns-rolebinding.yaml

apps/base/bind-external/records.yaml

@@ -0,0 +1,36 @@
+# Authoritative delegation records for acme.unkin.net. Without these the zone
+# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
+# primary pod IP), which is unroutable off-cluster and goes stale on
+# reschedule. DNSRecords must live in the same namespace as their BindZone.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-apex-ns
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  # "@" is the zone apex.
+  name: "@"
+  type: NS
+  ttl: 3600
+  values:
+    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
+    # child needs no glue of its own.
+    - acme-ns1.unkin.net.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-ns1-a
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  name: ns1
+  type: A
+  ttl: 3600
+  values:
+    # Public address of this cluster's external BIND, same target as
+    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
+    # NS name must still reach the zone.
+    - 103.216.191.185

apps/base/woodpecker/configmap_gocache-nginx.yaml

@@ -1,39 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: gocache-nginx
-  namespace: woodpecker
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: proxy
-data:
-  nginx.conf: |
-    worker_processes auto;
-    error_log /dev/stderr warn;
-    pid /tmp/nginx.pid;
-
-    events {
-      worker_connections 512;
-    }
-
-    # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}.
-    stream {
-      server {
-        listen 9090;
-
-        # The protocol has no authentication: anyone who can reach this port can
-        # write cache entries, which become code in every build that reads them.
-        # Loopback is the kubectl port-forward fallback; in a pod netns it is
-        # only these two containers.
-        allow 127.0.0.1/32;
-        allow 10.10.12.200/32;
-        allow 10.42.0.0/16;
-        deny all;
-
-        # A connect session lasts the whole build; the 10m default cuts long builds off.
-        proxy_timeout 2h;
-        proxy_connect_timeout 5s;
-        proxy_pass 127.0.0.1:9080;
-      }
-    }

apps/base/woodpecker/deployment_gocache.yaml

@@ -1,131 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    configmap.reloader.stakater.com/reload: "gocache-nginx"
-    secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert"
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: cache
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gocache
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: gocache
-        app.kubernetes.io/component: cache
-    spec:
-      serviceAccountName: default
-      automountServiceAccountToken: false
-      securityContext:
-        runAsNonRoot: true
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: go-cache-plugin
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0
-          imagePullPolicy: IfNotPresent
-          # Root flags must precede the subcommand; only --plugin belongs to serve.
-          args:
-            - --cache-dir=/var/cache/gocache
-            - --bucket=gocache
-            # Explicit region skips the GetBucketLocation probe, which RGW handles poorly.
-            - --region=us-east-1
-            - --s3-endpoint-url=https://s3.ceph.unkin.net
-            - --s3-path-style
-            - serve
-            - --plugin=9080
-          env:
-            - name: AWS_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_ACCESS_KEY_ID
-            - name: AWS_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_SECRET_ACCESS_KEY
-            # s3.ceph.unkin.net is served by the estate CA, not a public root.
-            - name: AWS_CA_BUNDLE
-              value: /etc/ssl/vault-ca/ca.crt
-          volumeMounts:
-            - name: cache
-              mountPath: /var/cache/gocache
-            - name: vault-ca
-              mountPath: /etc/ssl/vault-ca
-              readOnly: true
-          securityContext:
-            runAsUser: 65532
-            runAsGroup: 65532
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 200m
-              memory: 256Mi
-            limits:
-              cpu: "2"
-              memory: 2Gi
-        - name: nginx
-          image: docker.io/nginx:1.29.8-alpine
-          imagePullPolicy: IfNotPresent
-          # Bypass the image entrypoint: its config scripts write to a read-only rootfs.
-          command:
-            - nginx
-            - -g
-            - daemon off;
-          ports:
-            - containerPort: 9090
-              name: gocache
-              protocol: TCP
-          volumeMounts:
-            - name: nginx-config
-              mountPath: /etc/nginx/nginx.conf
-              subPath: nginx.conf
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-          securityContext:
-            runAsUser: 101
-            runAsGroup: 101
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 500m
-              memory: 128Mi
-      volumes:
-        # Staging cache in front of S3: losing it costs a repopulate, not data.
-        - name: cache
-          emptyDir:
-            sizeLimit: 20Gi
-        - name: nginx-config
-          configMap:
-            name: gocache-nginx
-        - name: tmp
-          emptyDir: {}
-        - name: vault-ca
-          secret:
-            secretName: vault-ca-cert
-            items:
-              - key: ca.crt
-                path: ca.crt

apps/base/woodpecker/kustomization.yaml

@@ -8,9 +8,6 @@ resources:
   - cnpg_backup.yaml
   - cnpg_pooler.yaml
   - gocache_bucket.yaml
-  - configmap_gocache-nginx.yaml
-  - deployment_gocache.yaml
-  - service_gocache.yaml
   - serviceaccount_arrproxy_ci.yaml
   - serviceaccount_autobackup_operator_ci.yaml
   - serviceaccount_ghp.yaml

apps/base/woodpecker/service_gocache.yaml

@@ -1,23 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    purelb.io/addresses: 198.18.200.11
-    purelb.io/service-group: common
-  labels:
-    app.kubernetes.io/name: gocache
-spec:
-  type: LoadBalancer
-  # Cluster SNATs off-node traffic to a node address, which would defeat the
-  # nginx allow rules; Local preserves the wireguard client IP.
-  externalTrafficPolicy: Local
-  selector:
-    app.kubernetes.io/name: gocache
-  ports:
-    - name: gocache
-      port: 9090
-      targetPort: gocache
-      protocol: TCP