diff --git a/apps/base/woodpecker/configmap_gocache-nginx.yaml b/apps/base/woodpecker/configmap_gocache-nginx.yaml new file mode 100644 index 0000000..ca1eaeb --- /dev/null +++ b/apps/base/woodpecker/configmap_gocache-nginx.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gocache-nginx + namespace: woodpecker + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: proxy +data: + nginx.conf: | + worker_processes auto; + error_log /dev/stderr warn; + pid /tmp/nginx.pid; + + events { + worker_connections 512; + } + + # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}. + stream { + server { + listen 9090; + + # The protocol has no authentication: anyone who can reach this port can + # write cache entries, which become code in every build that reads them. + # Loopback is the kubectl port-forward fallback; in a pod netns it is + # only these two containers. + allow 127.0.0.1/32; + allow 10.10.12.200/32; + allow 10.42.0.0/16; + deny all; + + # A connect session lasts the whole build; the 10m default cuts long builds off. + proxy_timeout 2h; + proxy_connect_timeout 5s; + proxy_pass 127.0.0.1:9080; + } + } diff --git a/apps/base/woodpecker/deployment_gocache.yaml b/apps/base/woodpecker/deployment_gocache.yaml new file mode 100644 index 0000000..dad3f9d --- /dev/null +++ b/apps/base/woodpecker/deployment_gocache.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gocache + namespace: woodpecker + annotations: + configmap.reloader.stakater.com/reload: "gocache-nginx" + secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert" + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: cache +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: gocache + template: + metadata: + labels: + app.kubernetes.io/name: gocache + app.kubernetes.io/component: cache + spec: + serviceAccountName: default + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: go-cache-plugin + image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0 + imagePullPolicy: IfNotPresent + # Root flags must precede the subcommand; only --plugin belongs to serve. + args: + - --cache-dir=/var/cache/gocache + - --bucket=gocache + # Explicit region skips the GetBucketLocation probe, which RGW handles poorly. + - --region=us-east-1 + - --s3-endpoint-url=https://s3.ceph.unkin.net + - --s3-path-style + - serve + - --plugin=9080 + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: gocache-s3 + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: gocache-s3 + key: AWS_SECRET_ACCESS_KEY + # s3.ceph.unkin.net is served by the estate CA, not a public root. + - name: AWS_CA_BUNDLE + value: /etc/ssl/vault-ca/ca.crt + volumeMounts: + - name: cache + mountPath: /var/cache/gocache + - name: vault-ca + mountPath: /etc/ssl/vault-ca + readOnly: true + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi + - name: nginx + image: docker.io/nginx:1.29.8-alpine + imagePullPolicy: IfNotPresent + # Bypass the image entrypoint: its config scripts write to a read-only rootfs. + command: + - nginx + - -g + - daemon off; + ports: + - containerPort: 9090 + name: gocache + protocol: TCP + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: tmp + mountPath: /tmp + securityContext: + runAsUser: 101 + runAsGroup: 101 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 500m + memory: 128Mi + volumes: + # Staging cache in front of S3: losing it costs a repopulate, not data. + - name: cache + emptyDir: + sizeLimit: 20Gi + - name: nginx-config + configMap: + name: gocache-nginx + - name: tmp + emptyDir: {} + - name: vault-ca + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt diff --git a/apps/base/woodpecker/kustomization.yaml b/apps/base/woodpecker/kustomization.yaml index ae394e2..ecddb7d 100644 --- a/apps/base/woodpecker/kustomization.yaml +++ b/apps/base/woodpecker/kustomization.yaml @@ -8,6 +8,9 @@ resources: - cnpg_backup.yaml - cnpg_pooler.yaml - gocache_bucket.yaml + - configmap_gocache-nginx.yaml + - deployment_gocache.yaml + - service_gocache.yaml - serviceaccount_arrproxy_ci.yaml - serviceaccount_autobackup_operator_ci.yaml - serviceaccount_ghp.yaml diff --git a/apps/base/woodpecker/service_gocache.yaml b/apps/base/woodpecker/service_gocache.yaml new file mode 100644 index 0000000..7cf0d7e --- /dev/null +++ b/apps/base/woodpecker/service_gocache.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: gocache + namespace: woodpecker + annotations: + purelb.io/addresses: 198.18.200.11 + purelb.io/service-group: common + labels: + app.kubernetes.io/name: gocache +spec: + type: LoadBalancer + # Cluster SNATs off-node traffic to a node address, which would defeat the + # nginx allow rules; Local preserves the wireguard client IP. + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: gocache + ports: + - name: gocache + port: 9090 + targetPort: gocache + protocol: TCP