Add ARC self-hosted runners for IUCLabs

.pre-commit-config.yaml

@@ -42,3 +42,9 @@ repos:
         entry: ci/validate-no-secrets.sh
         language: system
         pass_filenames: false
+      - id: runner_hooks
+        name: test arc runner job-started hook
+        entry: ci/test-runner-hooks.sh
+        language: system
+        pass_filenames: false
+        files: ^(apps/base/iuclabs-runners/job-started\.sh|ci/test-runner-hooks\.sh|ci/fixtures/runner-hooks/)

apps/base/arc-system/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - pdb.yaml

apps/base/arc-system/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: arc-system

apps/base/arc-system/pdb.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: arc-controller
+  namespace: arc-system
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gha-rs-controller
+      app.kubernetes.io/instance: arc

apps/base/iuclabs-runners/job-started.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -eu
+
+refuse() {
+  echo "job-started hook: refusing ${GITHUB_EVENT_NAME} job: $1" >&2
+  exit 1
+}
+
+case "${GITHUB_EVENT_NAME}" in
+  pull_request_target | workflow_run)
+    refuse "event not allowed on self-hosted runners"
+    ;;
+  pull_request)
+    same_repo=$(jq -r '.pull_request.head.repo.fork == false' "${GITHUB_EVENT_PATH}")
+    assoc=$(jq -r '.pull_request.author_association // "NONE"' "${GITHUB_EVENT_PATH}")
+    if [ "${same_repo}" != true ]; then
+      case "${assoc}" in
+        OWNER | MEMBER | COLLABORATOR) ;;
+        *) refuse "fork pull request from untrusted author (${assoc})" ;;
+      esac
+    fi
+    ;;
+esac

apps/base/iuclabs-runners/kustomization.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - networkpolicy.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configMapGenerator:
+  - name: runner-hooks
+    namespace: iuclabs-runners
+    files:
+      - job-started.sh

apps/base/iuclabs-runners/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: iuclabs-runners

apps/base/iuclabs-runners/networkpolicy.yaml

@@ -0,0 +1,67 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: runners
+  namespace: iuclabs-runners
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+              - 198.18.0.0/15
+      ports:
+        - protocol: TCP
+          port: 443
+    # artifactapi.k8s.syd1.au.unkin.net: traefik-internal LB, DNATed to its pods in-cluster
+    - to:
+        - ipBlock:
+            cidr: 198.18.200.4/32
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/instance: traefik-internal-traefik-system
+      ports:
+        - protocol: TCP
+          port: websecure
+    # vault.service.consul
+    - to:
+        - ipBlock:
+            cidr: 198.18.25.78/32
+        - ipBlock:
+            cidr: 198.18.26.197/32
+        - ipBlock:
+            cidr: 198.18.27.116/32
+        - ipBlock:
+            cidr: 198.18.28.248/32
+        - ipBlock:
+            cidr: 198.18.29.92/32
+      ports:
+        - protocol: TCP
+          port: 8200

apps/base/iuclabs-runners/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: iuclabs-runners
+spec:
+  allowedNamespaces:
+    - iuclabs-runners
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/iuclabs-runners/vaultstaticsecret.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: github-app
+  namespace: iuclabs-runners
+spec:
+  destination:
+    create: true
+    name: github-app
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/iuclabs-runners/default/github-app
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/overlays/au-syd1/arc-system/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/arc-system
+
+helmCharts:
+  - name: gha-runner-scale-set-controller
+    repo: oci://ghcr.io/actions/actions-runner-controller-charts
+    version: "0.15.1"
+    releaseName: arc
+    namespace: arc-system
+    valuesFile: values.yaml
+    includeCRDs: true

apps/overlays/au-syd1/arc-system/values.yaml

@@ -0,0 +1,34 @@
+---
+replicaCount: 2
+
+podSecurityContext:
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: DoNotSchedule
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: gha-rs-controller
+        app.kubernetes.io/instance: arc
+
+flags:
+  logLevel: info
+  logFormat: json

apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/iuclabs-runners
+
+helmCharts:
+  - name: gha-runner-scale-set
+    repo: oci://ghcr.io/actions/actions-runner-controller-charts
+    version: "0.15.1"
+    releaseName: iuclabs
+    namespace: iuclabs-runners
+    valuesFile: values.yaml

apps/overlays/au-syd1/iuclabs-runners/values.yaml

@@ -0,0 +1,55 @@
+---
+githubConfigUrl: https://github.com/IUCLabs
+githubConfigSecret: github-app
+runnerScaleSetName: iuclabs-k8s
+minRunners: 0
+maxRunners: 4
+
+controllerServiceAccount:
+  namespace: arc-system
+  name: arc-gha-rs-controller
+
+template:
+  spec:
+    automountServiceAccountToken: false
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 1001
+      runAsGroup: 1001
+      seccompProfile:
+        type: RuntimeDefault
+    topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: ScheduleAnyway
+        labelSelector:
+          matchLabels:
+            actions.github.com/scale-set-name: iuclabs-k8s
+    containers:
+      - name: runner
+        image: ghcr.io/actions/actions-runner:2.338.2
+        command: ["/home/runner/run.sh"]
+        env:
+          - name: ACTIONS_RUNNER_HOOK_JOB_STARTED
+            value: /hooks/job-started.sh
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+        resources:
+          requests:
+            cpu: 1
+            memory: 512Mi
+          limits:
+            cpu: 2
+            memory: 2Gi
+        volumeMounts:
+          - name: hooks
+            mountPath: /hooks
+            readOnly: true
+    volumes:
+      - name: hooks
+        configMap:
+          name: runner-hooks
+          defaultMode: 0555

argocd/applicationsets/platform.yaml

@@ -16,6 +16,7 @@ spec:
       - path: apps/overlays/*/bind-internal
       - path: apps/overlays/*/bind-external
       - path: apps/overlays/*/age-api
+      - path: apps/overlays/*/arc-system
       - path: apps/overlays/*/cattle-system
       - path: apps/overlays/*/cert-manager
       - path: apps/overlays/*/certificates
@@ -31,6 +32,7 @@ spec:
       - path: apps/overlays/*/grafana-system
       - path: apps/overlays/*/haproxy
       - path: apps/overlays/*/inteldeviceplugins-system
+      - path: apps/overlays/*/iuclabs-runners
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system
       - path: apps/overlays/*/kanidm

argocd/projects/platform.yaml

@@ -14,6 +14,8 @@ spec:
     - oci://ghcr.io/woodpecker-ci/helm/woodpecker
     - oci://ghcr.io/netbox-community/netbox-chart/netbox
     - oci://docker.gitea.com/charts
+    - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
+    - oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
   destinations:
     - namespace: '*-system'
       server: https://kubernetes.default.svc
@@ -45,6 +47,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'haproxy'
       server: https://kubernetes.default.svc
+    - namespace: 'iuclabs-runners'
+      server: https://kubernetes.default.svc
     - namespace: 'jfrog'
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'

ci/fixtures/runner-hooks/pr-deleted-fork.json

@@ -0,0 +1,8 @@
+{
+  "pull_request": {
+    "author_association": "NONE",
+    "head": {
+      "repo": null
+    }
+  }
+}

ci/fixtures/runner-hooks/pr-fork-none.json

@@ -0,0 +1,10 @@
+{
+  "pull_request": {
+    "author_association": "NONE",
+    "head": {
+      "repo": {
+        "fork": true
+      }
+    }
+  }
+}

ci/fixtures/runner-hooks/pr-member.json

@@ -0,0 +1,10 @@
+{
+  "pull_request": {
+    "author_association": "MEMBER",
+    "head": {
+      "repo": {
+        "fork": true
+      }
+    }
+  }
+}

ci/fixtures/runner-hooks/pr-same-repo.json

@@ -0,0 +1,10 @@
+{
+  "pull_request": {
+    "author_association": "CONTRIBUTOR",
+    "head": {
+      "repo": {
+        "fork": false
+      }
+    }
+  }
+}

ci/fixtures/runner-hooks/push.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

ci/test-runner-hooks.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+hook=apps/base/iuclabs-runners/job-started.sh
+fixtures=ci/fixtures/runner-hooks
+fail=0
+
+check() {
+  local want=$1 event=$2 fixture=$3 got=0
+  GITHUB_EVENT_NAME=$event GITHUB_EVENT_PATH=$fixtures/$fixture "$hook" >/dev/null 2>&1 || got=1
+  if [ "$got" -ne "$want" ]; then
+    echo "FAIL: $event $fixture exit=$got want=$want" >&2
+    fail=1
+  fi
+}
+
+check 0 push push.json
+check 0 pull_request pr-member.json
+check 0 pull_request pr-same-repo.json
+check 1 pull_request pr-fork-none.json
+check 1 pull_request pr-deleted-fork.json
+check 1 pull_request_target pr-member.json
+check 1 workflow_run push.json
+
+exit "$fail"