Drop the gocache serve Deployment and nginx sidecar (#499 )

`connect` mode gives the client no local cache, so every cache operation is a network round trip -- about 2.7x slower than pointing the plugin straight at S3. Developers and CI run the plugin in direct mode instead. - Remove the `go-cache-plugin serve` Deployment and its nginx stream-proxy ConfigMap - Remove the PureLB Service, freeing `198.18.200.11` - Keep the `gocache` bucket and `gocache-s3` Secret for direct-mode clients Reviewed-on: #499 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-09-27 11:20:59 +10:00
20 changed files with 165 additions and 397 deletions
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: dashboard-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: dashboard
-  type: CNAME
-  ttl: 600
-  values:
-    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
-    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
-    - lb1.unkin.net.
@@ -1,7 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - dashboard.yaml
-  - s3.yaml
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: s3-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: s3
-  type: CNAME
-  ttl: 600
-  values:
-    # radosgw S3 endpoint. Points at the Consul service for now; the real
-    # target will be changed later.
-    - radosgw.service.consul.
@@ -1,6 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - cname
@@ -2,14 +2,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

-# Individually-managed authoritative records live under <zone>/<type>/<record>.yaml.
-# DNSRecords must live in the same namespace as their BindZone (the operator
-# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
-# these sit alongside the zone in bind-internal, not in the app namespace.
 resources:
  - cluster.yaml
  - tsigkey.yaml
  - zones.yaml
-  - unkin-net
-  - ceph-unkin-net
+  - records.yaml
  - acls.yaml
@@ -0,0 +1,164 @@
+# Individually-managed authoritative records for the unkin.net zone.
+# DNSRecords must live in the same namespace as their BindZone (the operator
+# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
+# these sit alongside the zone in bind-internal, not in the app namespace.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  # "internal" in the name distinguishes this from the external DNS that
+  # Authentik will manage its own records from later.
+  name: identity-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: identity
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
+    # identity.unkin.net hostname there.
+    - 198.18.199.0
+---
+# PRODUCTION CUTOVER RECORD — intentionally commented out.
+# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
+# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
+# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
+# is the FINAL step of the forge migration — gated on the data migration (gitea
+# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
+# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
+# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
+# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
+# authority for unkin.net (or update the puppet record instead) before relying
+# on this CR at cutover.
+# ---
+# apiVersion: bind.unkin.net/v1alpha1
+# kind: DNSRecord
+# metadata:
+#   name: git-dns-internal
+#   namespace: bind-internal
+# spec:
+#   zoneRef: unkin-net
+#   name: git
+#   type: A
+#   ttl: 600
+#   values:
+#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
+#     - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: s3-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: s3
+  type: CNAME
+  ttl: 600
+  values:
+    # radosgw S3 endpoint. Points at the Consul service for now; the real
+    # target will be changed later.
+    - radosgw.service.consul.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: dashboard-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: dashboard
+  type: CNAME
+  ttl: 600
+  values:
+    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
+    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
+    - lb1.unkin.net.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: lb1-unkin-net
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: lb1
+  type: A
+  ttl: 600
+  values:
+    - 103.216.191.185
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: ghp-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: ghp
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: arrstack-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: arrstack
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
+    # arrstack.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: logviewer-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: logviewer
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the logviewer Gateway serves
+    # logviewer.unkin.net there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: cheeztv-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: cheeztv
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
+    # there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: watchstate-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: watchstate
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
+    # the watchstate.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: arrstack-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: arrstack
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
-    # arrstack.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: cheeztv-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: cheeztv
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
-    # there.
-    - 198.18.200.4
@@ -1,14 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: ghp-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: ghp
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
-    - 198.18.200.4
@@ -1,27 +0,0 @@
---
-# PRODUCTION CUTOVER RECORD — intentionally commented out.
-# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
-# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
-# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
-# is the FINAL step of the forge migration — gated on the data migration (gitea
-# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
-# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
-# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
-# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
-# authority for unkin.net (or update the puppet record instead) before relying
-# on this CR at cutover.
-# Uncomment this record AND its entry in kustomization.yaml to activate it.
-# ---
-# apiVersion: bind.unkin.net/v1alpha1
-# kind: DNSRecord
-# metadata:
-#   name: git-dns-internal
-#   namespace: bind-internal
-# spec:
-#   zoneRef: unkin-net
-#   name: git
-#   type: A
-#   ttl: 600
-#   values:
-#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
-#     - 198.18.200.4
@@ -1,17 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  # "internal" in the name distinguishes this from the external DNS that
-  # Authentik will manage its own records from later.
-  name: identity-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: identity
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
-    # identity.unkin.net hostname there.
-    - 198.18.199.0
@@ -1,15 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - arrstack.yaml
-  - cheeztv.yaml
-  # PRODUCTION CUTOVER RECORD — see git.yaml. Uncomment together with the
-  # record itself.
-  # - git.yaml
-  - ghp.yaml
-  - identity.yaml
-  - lb1.yaml
-  - logviewer.yaml
-  - watchstate.yaml
@@ -1,13 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: lb1-unkin-net
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: lb1
-  type: A
-  ttl: 600
-  values:
-    - 103.216.191.185
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: logviewer-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: logviewer
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the logviewer Gateway serves
-    # logviewer.unkin.net there.
-    - 198.18.200.4
@@ -1,15 +0,0 @@
---
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: watchstate-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: watchstate
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
-    # the watchstate.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0
@@ -1,6 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - a
@@ -1,39 +0,0 @@
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: gocache-nginx
-  namespace: woodpecker
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: proxy
-data:
-  nginx.conf: |
-    worker_processes auto;
-    error_log /dev/stderr warn;
-    pid /tmp/nginx.pid;
-
-    events {
-      worker_connections 512;
-    }
-
-    # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}.
-    stream {
-      server {
-        listen 9090;
-
-        # The protocol has no authentication: anyone who can reach this port can
-        # write cache entries, which become code in every build that reads them.
-        # Loopback is the kubectl port-forward fallback; in a pod netns it is
-        # only these two containers.
-        allow 127.0.0.1/32;
-        allow 10.10.12.200/32;
-        allow 10.42.0.0/16;
-        deny all;
-
-        # A connect session lasts the whole build; the 10m default cuts long builds off.
-        proxy_timeout 2h;
-        proxy_connect_timeout 5s;
-        proxy_pass 127.0.0.1:9080;
-      }
-    }
@@ -1,131 +0,0 @@
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    configmap.reloader.stakater.com/reload: "gocache-nginx"
-    secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert"
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: cache
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gocache
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: gocache
-        app.kubernetes.io/component: cache
-    spec:
-      serviceAccountName: default
-      automountServiceAccountToken: false
-      securityContext:
-        runAsNonRoot: true
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: go-cache-plugin
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0
-          imagePullPolicy: IfNotPresent
-          # Root flags must precede the subcommand; only --plugin belongs to serve.
-          args:
-            - --cache-dir=/var/cache/gocache
-            - --bucket=gocache
-            # Explicit region skips the GetBucketLocation probe, which RGW handles poorly.
-            - --region=us-east-1
-            - --s3-endpoint-url=https://s3.ceph.unkin.net
-            - --s3-path-style
-            - serve
-            - --plugin=9080
-          env:
-            - name: AWS_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_ACCESS_KEY_ID
-            - name: AWS_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_SECRET_ACCESS_KEY
-            # s3.ceph.unkin.net is served by the estate CA, not a public root.
-            - name: AWS_CA_BUNDLE
-              value: /etc/ssl/vault-ca/ca.crt
-          volumeMounts:
-            - name: cache
-              mountPath: /var/cache/gocache
-            - name: vault-ca
-              mountPath: /etc/ssl/vault-ca
-              readOnly: true
-          securityContext:
-            runAsUser: 65532
-            runAsGroup: 65532
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 200m
-              memory: 256Mi
-            limits:
-              cpu: "2"
-              memory: 2Gi
-        - name: nginx
-          image: docker.io/nginx:1.29.8-alpine
-          imagePullPolicy: IfNotPresent
-          # Bypass the image entrypoint: its config scripts write to a read-only rootfs.
-          command:
-            - nginx
-            - -g
-            - daemon off;
-          ports:
-            - containerPort: 9090
-              name: gocache
-              protocol: TCP
-          volumeMounts:
-            - name: nginx-config
-              mountPath: /etc/nginx/nginx.conf
-              subPath: nginx.conf
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-          securityContext:
-            runAsUser: 101
-            runAsGroup: 101
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 500m
-              memory: 128Mi
-      volumes:
-        # Staging cache in front of S3: losing it costs a repopulate, not data.
-        - name: cache
-          emptyDir:
-            sizeLimit: 20Gi
-        - name: nginx-config
-          configMap:
-            name: gocache-nginx
-        - name: tmp
-          emptyDir: {}
-        - name: vault-ca
-          secret:
-            secretName: vault-ca-cert
-            items:
-              - key: ca.crt
-                path: ca.crt
@@ -8,9 +8,6 @@ resources:
  - cnpg_backup.yaml
  - cnpg_pooler.yaml
  - gocache_bucket.yaml
-  - configmap_gocache-nginx.yaml
-  - deployment_gocache.yaml
-  - service_gocache.yaml
  - serviceaccount_arrproxy_ci.yaml
  - serviceaccount_autobackup_operator_ci.yaml
  - serviceaccount_ghp.yaml
@@ -1,23 +0,0 @@
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    purelb.io/addresses: 198.18.200.11
-    purelb.io/service-group: common
-  labels:
-    app.kubernetes.io/name: gocache
-spec:
-  type: LoadBalancer
-  # Cluster SNATs off-node traffic to a node address, which would defeat the
-  # nginx allow rules; Local preserves the wireguard client IP.
-  externalTrafficPolicy: Local
-  selector:
-    app.kubernetes.io/name: gocache
-  ports:
-    - name: gocache
-      port: 9090
-      targetPort: gocache
-      protocol: TCP