.pre-commit-config.yaml

@@ -42,3 +42,9 @@ repos:
         entry: ci/validate-no-secrets.sh
         language: system
         pass_filenames: false
+      - id: runner_hooks
+        name: test arc runner job-started hook
+        entry: ci/test-runner-hooks.sh
+        language: system
+        pass_filenames: false
+        files: ^(apps/base/iuclabs-runners/job-started\.sh|ci/test-runner-hooks\.sh|ci/fixtures/runner-hooks/)

apps/base/arc-system/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - pdb.yaml

apps/base/arc-system/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: arc-system

apps/base/arc-system/pdb.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: arc-controller
+  namespace: arc-system
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gha-rs-controller
+      app.kubernetes.io/instance: arc

apps/base/iuclabs-runners/components/iuclabs-k8s/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - scaleset.yaml
+  - networkpolicy.yaml

apps/base/iuclabs-runners/components/iuclabs-k8s/networkpolicy.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: egress-iuclabs-k8s
+  namespace: iuclabs-runners
+spec:
+  podSelector:
+    matchLabels:
+      runners.unkin.net/scale-set: iuclabs-k8s
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+              - 198.18.0.0/15
+              - 0.0.0.0/8
+              - 100.64.0.0/10
+              - 127.0.0.0/8
+              - 169.254.0.0/16
+              - 224.0.0.0/4
+              - 240.0.0.0/4
+      ports:
+        - protocol: TCP
+          port: 443
+    # vault.service.consul servers, from `dig vault.service.consul`
+    - to:
+        - ipBlock:
+            cidr: 198.18.25.78/32
+        - ipBlock:
+            cidr: 198.18.26.197/32
+        - ipBlock:
+            cidr: 198.18.27.116/32
+        - ipBlock:
+            cidr: 198.18.28.248/32
+        - ipBlock:
+            cidr: 198.18.29.92/32
+      ports:
+        - protocol: TCP
+          port: 8200

apps/base/iuclabs-runners/components/iuclabs-k8s/scaleset.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: actions.github.com/v1alpha1
+kind: AutoscalingRunnerSet
+metadata:
+  name: iuclabs-k8s
+  namespace: iuclabs-runners
+  labels:
+    app.kubernetes.io/version: "0.15.1"
+  annotations:
+    actions.github.com/cleanup-manager-role-name: arc-manager
+    actions.github.com/cleanup-manager-role-binding: arc-manager
+spec:
+  githubConfigUrl: https://github.com/IUCLabs
+  githubConfigSecret: github-app
+  runnerScaleSetName: iuclabs-k8s
+  minRunners: 0
+  maxRunners: 4
+  template:
+    metadata:
+      labels:
+        runners.unkin.net/scale-set: iuclabs-k8s
+    spec:
+      serviceAccountName: runner
+      automountServiceAccountToken: false
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1001
+        runAsGroup: 1001
+        seccompProfile:
+          type: RuntimeDefault
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              runners.unkin.net/scale-set: iuclabs-k8s
+      containers:
+        - name: runner
+          image: ghcr.io/actions/actions-runner:2.338.2
+          command: ["/home/runner/run.sh"]
+          env:
+            - name: ACTIONS_RUNNER_HOOK_JOB_STARTED
+              value: /hooks/job-started.sh
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 1
+              memory: 512Mi
+            limits:
+              cpu: 2
+              memory: 2Gi
+          volumeMounts:
+            - name: hooks
+              mountPath: /hooks
+              readOnly: true
+      volumes:
+        - name: hooks
+          configMap:
+            name: runner-hooks
+            defaultMode: 0555

apps/base/iuclabs-runners/job-started.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+
+event="${GITHUB_EVENT_NAME:-}"
+
+refuse() {
+  echo "job-started hook: refusing '${event}' job: $1" >&2
+  exit 1
+}
+
+[ -r "${GITHUB_EVENT_PATH:-}" ] || refuse "event payload missing"
+jq -e 'type == "object"' "${GITHUB_EVENT_PATH}" >/dev/null 2>&1 || refuse "event payload unparseable"
+
+case "${event}" in
+  push | workflow_dispatch | schedule | merge_group | release) ;;
+  pull_request)
+    trusted=$(jq -r '
+      .pull_request.head.repo.full_name as $head
+      | if $head == null then false
+        elif $head == .repository.full_name then true
+        else (.pull_request.author_association | IN("OWNER", "MEMBER", "COLLABORATOR"))
+        end' "${GITHUB_EVENT_PATH}") || refuse "event payload unreadable"
+    [ "${trusted}" = true ] || refuse "pull request from untrusted fork"
+    ;;
+  *) refuse "event not allowed on self-hosted runners" ;;
+esac

apps/base/iuclabs-runners/kustomization.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - networkpolicy.yaml
+  - rbac.yaml
+
+components:
+  - components/iuclabs-k8s
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configMapGenerator:
+  - name: runner-hooks
+    namespace: iuclabs-runners
+    files:
+      - job-started.sh

apps/base/iuclabs-runners/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: iuclabs-runners

apps/base/iuclabs-runners/networkpolicy.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny
+  namespace: iuclabs-runners
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-dns
+  namespace: iuclabs-runners
+spec:
+  podSelector: {}
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53

apps/base/iuclabs-runners/rbac.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: runner
+  namespace: iuclabs-runners
+automountServiceAccountToken: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: arc-manager
+  namespace: iuclabs-runners
+  finalizers:
+    - actions.github.com/cleanup-protection
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["create", "delete", "get"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["secrets", "serviceaccounts"]
+    verbs: ["create", "delete", "get", "list", "patch", "update"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["roles", "rolebindings"]
+    verbs: ["create", "delete", "get", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: arc-manager
+  namespace: iuclabs-runners
+  finalizers:
+    - actions.github.com/cleanup-protection
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: arc-manager
+subjects:
+  - kind: ServiceAccount
+    name: arc-gha-rs-controller
+    namespace: arc-system

apps/base/iuclabs-runners/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: iuclabs-runners
+spec:
+  allowedNamespaces:
+    - iuclabs-runners
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/iuclabs-runners/vaultstaticsecret.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: github-app
+  namespace: iuclabs-runners
+spec:
+  destination:
+    create: true
+    name: github-app
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/iuclabs-runners/default/github-app
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/overlays/au-syd1/arc-system/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/arc-system
+
+helmCharts:
+  - name: gha-runner-scale-set-controller
+    repo: oci://artifactapi.k8s.syd1.au.unkin.net/ghcr/actions/actions-runner-controller-charts
+    version: "0.15.1"
+    releaseName: arc
+    namespace: arc-system
+    valuesFile: values.yaml
+    includeCRDs: true

apps/overlays/au-syd1/arc-system/values.yaml

@@ -0,0 +1,34 @@
+---
+replicaCount: 2
+
+podSecurityContext:
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: DoNotSchedule
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: gha-rs-controller
+        app.kubernetes.io/instance: arc
+
+flags:
+  logLevel: info
+  logFormat: json

apps/overlays/au-syd1/iuclabs-runners/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/iuclabs-runners

argocd/applicationsets/platform.yaml

@@ -16,6 +16,7 @@ spec:
       - path: apps/overlays/*/bind-internal
       - path: apps/overlays/*/bind-external
       - path: apps/overlays/*/age-api
+      - path: apps/overlays/*/arc-system
       - path: apps/overlays/*/cattle-system
       - path: apps/overlays/*/cert-manager
       - path: apps/overlays/*/certificates
@@ -31,6 +32,7 @@ spec:
       - path: apps/overlays/*/grafana-system
       - path: apps/overlays/*/haproxy
       - path: apps/overlays/*/inteldeviceplugins-system
+      - path: apps/overlays/*/iuclabs-runners
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system
       - path: apps/overlays/*/kanidm

argocd/projects/platform.yaml

@@ -14,6 +14,7 @@ spec:
     - oci://ghcr.io/woodpecker-ci/helm/woodpecker
     - oci://ghcr.io/netbox-community/netbox-chart/netbox
     - oci://docker.gitea.com/charts
+    - oci://artifactapi.k8s.syd1.au.unkin.net/ghcr/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
   destinations:
     - namespace: '*-system'
       server: https://kubernetes.default.svc
@@ -45,6 +46,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'haproxy'
       server: https://kubernetes.default.svc
+    - namespace: 'iuclabs-runners'
+      server: https://kubernetes.default.svc
     - namespace: 'jfrog'
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'

ci/fixtures/runner-hooks/invalid.txt

@@ -0,0 +1 @@
+not json {

ci/fixtures/runner-hooks/issue-comment.json

@@ -0,0 +1,11 @@
+{
+  "comment": {
+    "author_association": "NONE"
+  },
+  "issue": {
+    "number": 1
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/pr-deleted-fork.json

@@ -0,0 +1,11 @@
+{
+  "pull_request": {
+    "author_association": "MEMBER",
+    "head": {
+      "repo": null
+    }
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/pr-fork-none.json

@@ -0,0 +1,14 @@
+{
+  "pull_request": {
+    "author_association": "NONE",
+    "head": {
+      "repo": {
+        "fork": true,
+        "full_name": "evil/app"
+      }
+    }
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/pr-fork-spoofed.json

@@ -0,0 +1,14 @@
+{
+  "pull_request": {
+    "author_association": "NONE",
+    "head": {
+      "repo": {
+        "fork": false,
+        "full_name": "evil/app"
+      }
+    }
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/pr-member.json

@@ -0,0 +1,14 @@
+{
+  "pull_request": {
+    "author_association": "MEMBER",
+    "head": {
+      "repo": {
+        "fork": true,
+        "full_name": "someone/app"
+      }
+    }
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/pr-same-repo.json

@@ -0,0 +1,14 @@
+{
+  "pull_request": {
+    "author_association": "CONTRIBUTOR",
+    "head": {
+      "repo": {
+        "fork": false,
+        "full_name": "IUCLabs/app"
+      }
+    }
+  },
+  "repository": {
+    "full_name": "IUCLabs/app"
+  }
+}

ci/fixtures/runner-hooks/push.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

ci/test-runner-hooks.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+hook=apps/base/iuclabs-runners/job-started.sh
+fixtures=ci/fixtures/runner-hooks
+fail=0
+
+check() {
+  local want=$1 event=$2 fixture=$3 got=0
+  local -a env=(-u GITHUB_EVENT_NAME -u GITHUB_EVENT_PATH)
+  [ "$event" = - ] || env+=("GITHUB_EVENT_NAME=$event")
+  [ "$fixture" = - ] || env+=("GITHUB_EVENT_PATH=$fixtures/$fixture")
+  env "${env[@]}" "$hook" >/dev/null 2>&1 || got=1
+  if [ "$got" -ne "$want" ]; then
+    echo "FAIL: $event $fixture exit=$got want=$want" >&2
+    fail=1
+  fi
+}
+
+for e in push workflow_dispatch schedule merge_group release; do
+  check 0 "$e" push.json
+done
+check 0 pull_request pr-member.json
+check 0 pull_request pr-same-repo.json
+check 1 pull_request pr-fork-none.json
+check 1 pull_request pr-fork-spoofed.json
+check 1 pull_request pr-deleted-fork.json
+check 1 pull_request push.json
+for e in pull_request_target workflow_run pull_request_review pull_request_review_comment issue_comment issues discussion discussion_comment create delete unknown_event; do
+  check 1 "$e" pr-member.json
+done
+check 1 pull_request_review pr-fork-none.json
+check 1 issue_comment issue-comment.json
+check 1 push -
+check 1 push missing.json
+check 1 push invalid.txt
+check 1 - push.json
+
+exit "$fail"