From 5cbbb9ea212b7ea5a14246d65b163c41e5e83b80 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Tue, 10 Nov 2026 22:20:35 +1100 Subject: [PATCH] Add ARC self-hosted runners for IUCLabs --- .pre-commit-config.yaml | 6 ++ apps/base/arc-system/kustomization.yaml | 7 ++ apps/base/arc-system/namespace.yaml | 5 ++ apps/base/arc-system/pdb.yaml | 12 ++++ apps/base/iuclabs-runners/job-started.sh | 23 +++++++ apps/base/iuclabs-runners/kustomization.yaml | 18 +++++ apps/base/iuclabs-runners/namespace.yaml | 5 ++ apps/base/iuclabs-runners/networkpolicy.yaml | 67 +++++++++++++++++++ apps/base/iuclabs-runners/vaultauth.yaml | 18 +++++ .../iuclabs-runners/vaultstaticsecret.yaml | 17 +++++ .../au-syd1/arc-system/kustomization.yaml | 15 +++++ apps/overlays/au-syd1/arc-system/values.yaml | 34 ++++++++++ .../iuclabs-runners/kustomization.yaml | 14 ++++ .../au-syd1/iuclabs-runners/values.yaml | 55 +++++++++++++++ argocd/applicationsets/platform.yaml | 2 + argocd/projects/platform.yaml | 4 ++ ci/fixtures/runner-hooks/pr-deleted-fork.json | 8 +++ ci/fixtures/runner-hooks/pr-fork-none.json | 10 +++ ci/fixtures/runner-hooks/pr-member.json | 10 +++ ci/fixtures/runner-hooks/pr-same-repo.json | 10 +++ ci/fixtures/runner-hooks/push.json | 3 + ci/test-runner-hooks.sh | 25 +++++++ 22 files changed, 368 insertions(+) create mode 100644 apps/base/arc-system/kustomization.yaml create mode 100644 apps/base/arc-system/namespace.yaml create mode 100644 apps/base/arc-system/pdb.yaml create mode 100755 apps/base/iuclabs-runners/job-started.sh create mode 100644 apps/base/iuclabs-runners/kustomization.yaml create mode 100644 apps/base/iuclabs-runners/namespace.yaml create mode 100644 apps/base/iuclabs-runners/networkpolicy.yaml create mode 100644 apps/base/iuclabs-runners/vaultauth.yaml create mode 100644 apps/base/iuclabs-runners/vaultstaticsecret.yaml create mode 100644 apps/overlays/au-syd1/arc-system/kustomization.yaml create mode 100644 apps/overlays/au-syd1/arc-system/values.yaml create mode 100644 apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml create mode 100644 apps/overlays/au-syd1/iuclabs-runners/values.yaml create mode 100644 ci/fixtures/runner-hooks/pr-deleted-fork.json create mode 100644 ci/fixtures/runner-hooks/pr-fork-none.json create mode 100644 ci/fixtures/runner-hooks/pr-member.json create mode 100644 ci/fixtures/runner-hooks/pr-same-repo.json create mode 100644 ci/fixtures/runner-hooks/push.json create mode 100755 ci/test-runner-hooks.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a3fb06..1ea6adc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,3 +42,9 @@ repos: entry: ci/validate-no-secrets.sh language: system pass_filenames: false + - id: runner_hooks + name: test arc runner job-started hook + entry: ci/test-runner-hooks.sh + language: system + pass_filenames: false + files: ^(apps/base/iuclabs-runners/job-started\.sh|ci/test-runner-hooks\.sh|ci/fixtures/runner-hooks/) diff --git a/apps/base/arc-system/kustomization.yaml b/apps/base/arc-system/kustomization.yaml new file mode 100644 index 0000000..a26b1fa --- /dev/null +++ b/apps/base/arc-system/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - pdb.yaml diff --git a/apps/base/arc-system/namespace.yaml b/apps/base/arc-system/namespace.yaml new file mode 100644 index 0000000..bc771a9 --- /dev/null +++ b/apps/base/arc-system/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: arc-system diff --git a/apps/base/arc-system/pdb.yaml b/apps/base/arc-system/pdb.yaml new file mode 100644 index 0000000..498e168 --- /dev/null +++ b/apps/base/arc-system/pdb.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: arc-controller + namespace: arc-system +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: gha-rs-controller + app.kubernetes.io/instance: arc diff --git a/apps/base/iuclabs-runners/job-started.sh b/apps/base/iuclabs-runners/job-started.sh new file mode 100755 index 0000000..c7203cd --- /dev/null +++ b/apps/base/iuclabs-runners/job-started.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +refuse() { + echo "job-started hook: refusing ${GITHUB_EVENT_NAME} job: $1" >&2 + exit 1 +} + +case "${GITHUB_EVENT_NAME}" in + pull_request_target | workflow_run) + refuse "event not allowed on self-hosted runners" + ;; + pull_request) + same_repo=$(jq -r '.pull_request.head.repo.fork == false' "${GITHUB_EVENT_PATH}") + assoc=$(jq -r '.pull_request.author_association // "NONE"' "${GITHUB_EVENT_PATH}") + if [ "${same_repo}" != true ]; then + case "${assoc}" in + OWNER | MEMBER | COLLABORATOR) ;; + *) refuse "fork pull request from untrusted author (${assoc})" ;; + esac + fi + ;; +esac diff --git a/apps/base/iuclabs-runners/kustomization.yaml b/apps/base/iuclabs-runners/kustomization.yaml new file mode 100644 index 0000000..8d0cf12 --- /dev/null +++ b/apps/base/iuclabs-runners/kustomization.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - networkpolicy.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - name: runner-hooks + namespace: iuclabs-runners + files: + - job-started.sh diff --git a/apps/base/iuclabs-runners/namespace.yaml b/apps/base/iuclabs-runners/namespace.yaml new file mode 100644 index 0000000..fa25861 --- /dev/null +++ b/apps/base/iuclabs-runners/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: iuclabs-runners diff --git a/apps/base/iuclabs-runners/networkpolicy.yaml b/apps/base/iuclabs-runners/networkpolicy.yaml new file mode 100644 index 0000000..44329a1 --- /dev/null +++ b/apps/base/iuclabs-runners/networkpolicy.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: runners + namespace: iuclabs-runners +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 198.18.0.0/15 + ports: + - protocol: TCP + port: 443 + # artifactapi.k8s.syd1.au.unkin.net: traefik-internal LB, DNATed to its pods in-cluster + - to: + - ipBlock: + cidr: 198.18.200.4/32 + ports: + - protocol: TCP + port: 443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik-system + podSelector: + matchLabels: + app.kubernetes.io/instance: traefik-internal-traefik-system + ports: + - protocol: TCP + port: websecure + # vault.service.consul + - to: + - ipBlock: + cidr: 198.18.25.78/32 + - ipBlock: + cidr: 198.18.26.197/32 + - ipBlock: + cidr: 198.18.27.116/32 + - ipBlock: + cidr: 198.18.28.248/32 + - ipBlock: + cidr: 198.18.29.92/32 + ports: + - protocol: TCP + port: 8200 diff --git a/apps/base/iuclabs-runners/vaultauth.yaml b/apps/base/iuclabs-runners/vaultauth.yaml new file mode 100644 index 0000000..066605e --- /dev/null +++ b/apps/base/iuclabs-runners/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: iuclabs-runners +spec: + allowedNamespaces: + - iuclabs-runners + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/iuclabs-runners/vaultstaticsecret.yaml b/apps/base/iuclabs-runners/vaultstaticsecret.yaml new file mode 100644 index 0000000..bad4ff6 --- /dev/null +++ b/apps/base/iuclabs-runners/vaultstaticsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: github-app + namespace: iuclabs-runners +spec: + destination: + create: true + name: github-app + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/iuclabs-runners/default/github-app + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/overlays/au-syd1/arc-system/kustomization.yaml b/apps/overlays/au-syd1/arc-system/kustomization.yaml new file mode 100644 index 0000000..270f056 --- /dev/null +++ b/apps/overlays/au-syd1/arc-system/kustomization.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/arc-system + +helmCharts: + - name: gha-runner-scale-set-controller + repo: oci://ghcr.io/actions/actions-runner-controller-charts + version: "0.15.1" + releaseName: arc + namespace: arc-system + valuesFile: values.yaml + includeCRDs: true diff --git a/apps/overlays/au-syd1/arc-system/values.yaml b/apps/overlays/au-syd1/arc-system/values.yaml new file mode 100644 index 0000000..32bbd5e --- /dev/null +++ b/apps/overlays/au-syd1/arc-system/values.yaml @@ -0,0 +1,34 @@ +--- +replicaCount: 2 + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: gha-rs-controller + app.kubernetes.io/instance: arc + +flags: + logLevel: info + logFormat: json diff --git a/apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml b/apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml new file mode 100644 index 0000000..c35c13f --- /dev/null +++ b/apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/iuclabs-runners + +helmCharts: + - name: gha-runner-scale-set + repo: oci://ghcr.io/actions/actions-runner-controller-charts + version: "0.15.1" + releaseName: iuclabs + namespace: iuclabs-runners + valuesFile: values.yaml diff --git a/apps/overlays/au-syd1/iuclabs-runners/values.yaml b/apps/overlays/au-syd1/iuclabs-runners/values.yaml new file mode 100644 index 0000000..c42f765 --- /dev/null +++ b/apps/overlays/au-syd1/iuclabs-runners/values.yaml @@ -0,0 +1,55 @@ +--- +githubConfigUrl: https://github.com/IUCLabs +githubConfigSecret: github-app +runnerScaleSetName: iuclabs-k8s +minRunners: 0 +maxRunners: 4 + +controllerServiceAccount: + namespace: arc-system + name: arc-gha-rs-controller + +template: + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + seccompProfile: + type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + actions.github.com/scale-set-name: iuclabs-k8s + containers: + - name: runner + image: ghcr.io/actions/actions-runner:2.338.2 + command: ["/home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_HOOK_JOB_STARTED + value: /hooks/job-started.sh + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 1 + memory: 512Mi + limits: + cpu: 2 + memory: 2Gi + volumeMounts: + - name: hooks + mountPath: /hooks + readOnly: true + volumes: + - name: hooks + configMap: + name: runner-hooks + defaultMode: 0555 diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index b0c9d3c..2376860 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -16,6 +16,7 @@ spec: - path: apps/overlays/*/bind-internal - path: apps/overlays/*/bind-external - path: apps/overlays/*/age-api + - path: apps/overlays/*/arc-system - path: apps/overlays/*/cattle-system - path: apps/overlays/*/cert-manager - path: apps/overlays/*/certificates @@ -31,6 +32,7 @@ spec: - path: apps/overlays/*/grafana-system - path: apps/overlays/*/haproxy - path: apps/overlays/*/inteldeviceplugins-system + - path: apps/overlays/*/iuclabs-runners - path: apps/overlays/*/jfrog - path: apps/overlays/*/k8up-system - path: apps/overlays/*/kanidm diff --git a/argocd/projects/platform.yaml b/argocd/projects/platform.yaml index a7e97fa..8140509 100644 --- a/argocd/projects/platform.yaml +++ b/argocd/projects/platform.yaml @@ -14,6 +14,8 @@ spec: - oci://ghcr.io/woodpecker-ci/helm/woodpecker - oci://ghcr.io/netbox-community/netbox-chart/netbox - oci://docker.gitea.com/charts + - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set destinations: - namespace: '*-system' server: https://kubernetes.default.svc @@ -45,6 +47,8 @@ spec: server: https://kubernetes.default.svc - namespace: 'haproxy' server: https://kubernetes.default.svc + - namespace: 'iuclabs-runners' + server: https://kubernetes.default.svc - namespace: 'jfrog' server: https://kubernetes.default.svc - namespace: 'kanidm' diff --git a/ci/fixtures/runner-hooks/pr-deleted-fork.json b/ci/fixtures/runner-hooks/pr-deleted-fork.json new file mode 100644 index 0000000..a602183 --- /dev/null +++ b/ci/fixtures/runner-hooks/pr-deleted-fork.json @@ -0,0 +1,8 @@ +{ + "pull_request": { + "author_association": "NONE", + "head": { + "repo": null + } + } +} diff --git a/ci/fixtures/runner-hooks/pr-fork-none.json b/ci/fixtures/runner-hooks/pr-fork-none.json new file mode 100644 index 0000000..1c4104b --- /dev/null +++ b/ci/fixtures/runner-hooks/pr-fork-none.json @@ -0,0 +1,10 @@ +{ + "pull_request": { + "author_association": "NONE", + "head": { + "repo": { + "fork": true + } + } + } +} diff --git a/ci/fixtures/runner-hooks/pr-member.json b/ci/fixtures/runner-hooks/pr-member.json new file mode 100644 index 0000000..50ac080 --- /dev/null +++ b/ci/fixtures/runner-hooks/pr-member.json @@ -0,0 +1,10 @@ +{ + "pull_request": { + "author_association": "MEMBER", + "head": { + "repo": { + "fork": true + } + } + } +} diff --git a/ci/fixtures/runner-hooks/pr-same-repo.json b/ci/fixtures/runner-hooks/pr-same-repo.json new file mode 100644 index 0000000..81312c5 --- /dev/null +++ b/ci/fixtures/runner-hooks/pr-same-repo.json @@ -0,0 +1,10 @@ +{ + "pull_request": { + "author_association": "CONTRIBUTOR", + "head": { + "repo": { + "fork": false + } + } + } +} diff --git a/ci/fixtures/runner-hooks/push.json b/ci/fixtures/runner-hooks/push.json new file mode 100644 index 0000000..ccb4cb1 --- /dev/null +++ b/ci/fixtures/runner-hooks/push.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/heads/main" +} diff --git a/ci/test-runner-hooks.sh b/ci/test-runner-hooks.sh new file mode 100755 index 0000000..0189133 --- /dev/null +++ b/ci/test-runner-hooks.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +hook=apps/base/iuclabs-runners/job-started.sh +fixtures=ci/fixtures/runner-hooks +fail=0 + +check() { + local want=$1 event=$2 fixture=$3 got=0 + GITHUB_EVENT_NAME=$event GITHUB_EVENT_PATH=$fixtures/$fixture "$hook" >/dev/null 2>&1 || got=1 + if [ "$got" -ne "$want" ]; then + echo "FAIL: $event $fixture exit=$got want=$want" >&2 + fail=1 + fi +} + +check 0 push push.json +check 0 pull_request pr-member.json +check 0 pull_request pr-same-repo.json +check 1 pull_request pr-fork-none.json +check 1 pull_request pr-deleted-fork.json +check 1 pull_request_target pr-member.json +check 1 workflow_run push.json + +exit "$fail"