Add ARC self-hosted runners for IUCLabs (#579)
IUCLabs public repos need self-hosted CI capacity, but anyone can open PRs there, so runners must only execute trusted code and must not reach internal services. - deploy the ARC controller chart in arc-system, pulled through artifactapi - keep shared runner resources (namespace, manager Role/RoleBinding, VSO secret, hooks) in the iuclabs-runners base - package each scale set as a kustomize component (AutoscalingRunnerSet + its HTTPS/Vault egress policy), starting with iuclabs-k8s - allow only push/dispatch/schedule/merge_group/release and trusted pull_request jobs via a job-started hook, tested in pre-commit - default-deny the namespace and allow DNS Depends on the github-app secret existing in Vault. Reviewed-on: #579 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #579.
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- pdb.yaml
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: arc-system
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: arc-controller
|
||||
namespace: arc-system
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gha-rs-controller
|
||||
app.kubernetes.io/instance: arc
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
|
||||
resources:
|
||||
- scaleset.yaml
|
||||
- networkpolicy.yaml
|
||||
@@ -0,0 +1,45 @@
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: egress-iuclabs-k8s
|
||||
namespace: iuclabs-runners
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
runners.unkin.net/scale-set: iuclabs-k8s
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 10.0.0.0/8
|
||||
- 172.16.0.0/12
|
||||
- 192.168.0.0/16
|
||||
- 198.18.0.0/15
|
||||
- 0.0.0.0/8
|
||||
- 100.64.0.0/10
|
||||
- 127.0.0.0/8
|
||||
- 169.254.0.0/16
|
||||
- 224.0.0.0/4
|
||||
- 240.0.0.0/4
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 443
|
||||
# vault.service.consul servers, from `dig vault.service.consul`
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 198.18.25.78/32
|
||||
- ipBlock:
|
||||
cidr: 198.18.26.197/32
|
||||
- ipBlock:
|
||||
cidr: 198.18.27.116/32
|
||||
- ipBlock:
|
||||
cidr: 198.18.28.248/32
|
||||
- ipBlock:
|
||||
cidr: 198.18.29.92/32
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8200
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
apiVersion: actions.github.com/v1alpha1
|
||||
kind: AutoscalingRunnerSet
|
||||
metadata:
|
||||
name: iuclabs-k8s
|
||||
namespace: iuclabs-runners
|
||||
labels:
|
||||
app.kubernetes.io/version: "0.15.1"
|
||||
annotations:
|
||||
actions.github.com/cleanup-manager-role-name: arc-manager
|
||||
actions.github.com/cleanup-manager-role-binding: arc-manager
|
||||
spec:
|
||||
githubConfigUrl: https://github.com/IUCLabs
|
||||
githubConfigSecret: github-app
|
||||
runnerScaleSetName: iuclabs-k8s
|
||||
minRunners: 0
|
||||
maxRunners: 4
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
runners.unkin.net/scale-set: iuclabs-k8s
|
||||
spec:
|
||||
serviceAccountName: runner
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
runners.unkin.net/scale-set: iuclabs-k8s
|
||||
containers:
|
||||
- name: runner
|
||||
image: ghcr.io/actions/actions-runner:2.338.2
|
||||
command: ["/home/runner/run.sh"]
|
||||
env:
|
||||
- name: ACTIONS_RUNNER_HOOK_JOB_STARTED
|
||||
value: /hooks/job-started.sh
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 2
|
||||
memory: 2Gi
|
||||
volumeMounts:
|
||||
- name: hooks
|
||||
mountPath: /hooks
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: hooks
|
||||
configMap:
|
||||
name: runner-hooks
|
||||
defaultMode: 0555
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
event="${GITHUB_EVENT_NAME:-}"
|
||||
|
||||
refuse() {
|
||||
echo "job-started hook: refusing '${event}' job: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -r "${GITHUB_EVENT_PATH:-}" ] || refuse "event payload missing"
|
||||
jq -e 'type == "object"' "${GITHUB_EVENT_PATH}" >/dev/null 2>&1 || refuse "event payload unparseable"
|
||||
|
||||
case "${event}" in
|
||||
push | workflow_dispatch | schedule | merge_group | release) ;;
|
||||
pull_request)
|
||||
trusted=$(jq -r '
|
||||
.pull_request.head.repo.full_name as $head
|
||||
| if $head == null then false
|
||||
elif $head == .repository.full_name then true
|
||||
else (.pull_request.author_association | IN("OWNER", "MEMBER", "COLLABORATOR"))
|
||||
end' "${GITHUB_EVENT_PATH}") || refuse "event payload unreadable"
|
||||
[ "${trusted}" = true ] || refuse "pull request from untrusted fork"
|
||||
;;
|
||||
*) refuse "event not allowed on self-hosted runners" ;;
|
||||
esac
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- vaultauth.yaml
|
||||
- vaultstaticsecret.yaml
|
||||
- networkpolicy.yaml
|
||||
- rbac.yaml
|
||||
|
||||
components:
|
||||
- components/iuclabs-k8s
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
configMapGenerator:
|
||||
- name: runner-hooks
|
||||
namespace: iuclabs-runners
|
||||
files:
|
||||
- job-started.sh
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: iuclabs-runners
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: iuclabs-runners
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-dns
|
||||
namespace: iuclabs-runners
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: runner
|
||||
namespace: iuclabs-runners
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: arc-manager
|
||||
namespace: iuclabs-runners
|
||||
finalizers:
|
||||
- actions.github.com/cleanup-protection
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["create", "delete", "get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/status"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets", "serviceaccounts"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "update"]
|
||||
- apiGroups: ["rbac.authorization.k8s.io"]
|
||||
resources: ["roles", "rolebindings"]
|
||||
verbs: ["create", "delete", "get", "patch", "update"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: arc-manager
|
||||
namespace: iuclabs-runners
|
||||
finalizers:
|
||||
- actions.github.com/cleanup-protection
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: arc-manager
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: arc-gha-rs-controller
|
||||
namespace: arc-system
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: default
|
||||
namespace: iuclabs-runners
|
||||
spec:
|
||||
allowedNamespaces:
|
||||
- iuclabs-runners
|
||||
kubernetes:
|
||||
audiences:
|
||||
- vault
|
||||
role: default
|
||||
serviceAccount: default
|
||||
tokenExpirationSeconds: 600
|
||||
method: kubernetes
|
||||
mount: k8s/au/syd1
|
||||
vaultConnectionRef: vso-system/default
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: github-app
|
||||
namespace: iuclabs-runners
|
||||
spec:
|
||||
destination:
|
||||
create: true
|
||||
name: github-app
|
||||
overwrite: true
|
||||
hmacSecretData: true
|
||||
mount: kv
|
||||
path: kubernetes/namespace/iuclabs-runners/default/github-app
|
||||
refreshAfter: 5m
|
||||
type: kv-v2
|
||||
vaultAuthRef: default
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- ../../../base/arc-system
|
||||
|
||||
helmCharts:
|
||||
- name: gha-runner-scale-set-controller
|
||||
repo: oci://artifactapi.k8s.syd1.au.unkin.net/ghcr/actions/actions-runner-controller-charts
|
||||
version: "0.15.1"
|
||||
releaseName: arc
|
||||
namespace: arc-system
|
||||
valuesFile: values.yaml
|
||||
includeCRDs: true
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
replicaCount: 2
|
||||
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gha-rs-controller
|
||||
app.kubernetes.io/instance: arc
|
||||
|
||||
flags:
|
||||
logLevel: info
|
||||
logFormat: json
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- ../../../base/iuclabs-runners
|
||||
Reference in New Issue
Block a user