Add gocache serve Deployment for the shared Go build cache
go-cache-plugin serve binds 127.0.0.1 only, so a Service cannot reach it directly and CI/developer builds have no way to use the S3-backed cache. - Run go-cache-plugin serve against the gocache RGW bucket, path-style, with an explicit region to skip the GetBucketLocation probe - Add an nginx sidecar stream-proxying 9090 to the loopback plugin port - Restrict the listener to the workstation and pod CIDRs: GOCACHEPROG is unauthenticated and a poisoned entry runs in every consuming build - Stage the cache on an emptyDir; loss costs a repopulate from S3
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: gocache-nginx
|
||||
namespace: woodpecker
|
||||
labels:
|
||||
app.kubernetes.io/name: gocache
|
||||
app.kubernetes.io/component: proxy
|
||||
data:
|
||||
nginx.conf: |
|
||||
worker_processes auto;
|
||||
error_log /dev/stderr warn;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 512;
|
||||
}
|
||||
|
||||
# GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}.
|
||||
stream {
|
||||
server {
|
||||
listen 9090;
|
||||
|
||||
# The protocol has no authentication: anyone who can reach this port can
|
||||
# write cache entries, which become code in every build that reads them.
|
||||
allow 127.0.0.1/32;
|
||||
allow 10.10.12.200/32;
|
||||
allow 10.42.0.0/16;
|
||||
deny all;
|
||||
|
||||
# A connect session lasts the whole build; the 10m default cuts long builds off.
|
||||
proxy_timeout 2h;
|
||||
proxy_connect_timeout 5s;
|
||||
proxy_pass 127.0.0.1:9080;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: gocache
|
||||
namespace: woodpecker
|
||||
annotations:
|
||||
configmap.reloader.stakater.com/reload: "gocache-nginx"
|
||||
secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert"
|
||||
labels:
|
||||
app.kubernetes.io/name: gocache
|
||||
app.kubernetes.io/component: cache
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gocache
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: gocache
|
||||
app.kubernetes.io/component: cache
|
||||
spec:
|
||||
serviceAccountName: default
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
fsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: go-cache-plugin
|
||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
# Root flags must precede the subcommand; only --plugin belongs to serve.
|
||||
args:
|
||||
- --cache-dir=/var/cache/gocache
|
||||
- --bucket=gocache
|
||||
# Explicit region skips the GetBucketLocation probe, which RGW handles poorly.
|
||||
- --region=us-east-1
|
||||
- --s3-endpoint-url=https://s3.ceph.unkin.net
|
||||
- --s3-path-style
|
||||
- serve
|
||||
- --plugin=9080
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gocache-s3
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gocache-s3
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
# s3.ceph.unkin.net is served by the estate CA, not a public root.
|
||||
- name: AWS_CA_BUNDLE
|
||||
value: /etc/ssl/vault-ca/ca.crt
|
||||
volumeMounts:
|
||||
- name: cache
|
||||
mountPath: /var/cache/gocache
|
||||
- name: vault-ca
|
||||
mountPath: /etc/ssl/vault-ca
|
||||
readOnly: true
|
||||
securityContext:
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
- name: nginx
|
||||
image: docker.io/nginx:1.29.8-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
# Bypass the image entrypoint: its config scripts write to a read-only rootfs.
|
||||
command:
|
||||
- nginx
|
||||
- -g
|
||||
- daemon off;
|
||||
ports:
|
||||
- containerPort: 9090
|
||||
name: gocache
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: nginx-config
|
||||
mountPath: /etc/nginx/nginx.conf
|
||||
subPath: nginx.conf
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
securityContext:
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 128Mi
|
||||
volumes:
|
||||
# Staging cache in front of S3: losing it costs a repopulate, not data.
|
||||
- name: cache
|
||||
emptyDir:
|
||||
sizeLimit: 20Gi
|
||||
- name: nginx-config
|
||||
configMap:
|
||||
name: gocache-nginx
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: vault-ca
|
||||
secret:
|
||||
secretName: vault-ca-cert
|
||||
items:
|
||||
- key: ca.crt
|
||||
path: ca.crt
|
||||
@@ -8,6 +8,9 @@ resources:
|
||||
- cnpg_backup.yaml
|
||||
- cnpg_pooler.yaml
|
||||
- gocache_bucket.yaml
|
||||
- configmap_gocache-nginx.yaml
|
||||
- deployment_gocache.yaml
|
||||
- service_gocache.yaml
|
||||
- serviceaccount_arrproxy_ci.yaml
|
||||
- serviceaccount_autobackup_operator_ci.yaml
|
||||
- serviceaccount_ghp.yaml
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gocache
|
||||
namespace: woodpecker
|
||||
labels:
|
||||
app.kubernetes.io/name: gocache
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: gocache
|
||||
ports:
|
||||
- name: gocache
|
||||
port: 9090
|
||||
targetPort: gocache
|
||||
protocol: TCP
|
||||
Reference in New Issue
Block a user