From 7aec9a902179afec90ab0d8c4768a7d5a7f3ddc8 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 7 Sep 2026 23:05:39 +1000 Subject: [PATCH] artifactapi: restore combine-certs + PROVIDER_CA_FILES on oauth2-proxy (#458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Fix-forward companion to the #457 rollback. This is NOT the current outage fix — see below.** ## The actual outage The UI is 503 because the Authentik application slug `artifactapi` **does not exist**. OIDC discovery 404s, so oauth2-proxy exits at startup, the Service has no ready endpoints, and Traefik answers `no available server`. ``` identity.unkin.net /application/o/artifactapi/… 404 identity.k8s.syd1.au.unkin.net /application/o/artifactapi/… 404 identity.unkin.net /application/o/repospawner/… 200 identity.unkin.net /application/o/argocd/… 200 ``` Root cause is upstream in **terraform-authentik**: `ci/woodpecker/push/apply` on main HEAD `4e16401` **failed**. That apply has to succeed before any argocd-apps change can help. **This PR does not fix that.** ## What this PR does fix #456 dropped the `combine-certs` initContainer and `OAUTH2_PROXY_PROVIDER_CA_FILES`, reasoning that `identity.unkin.net` serves a publicly trusted Let's Encrypt cert and so needs no internal CA. That holds for the browser redirect but not for oauth2-proxy's own back-channel discovery/token calls. artifactapi is the **only one of six** oauth2-proxies in the estate without it: | app | issuer host | `PROVIDER_CA_FILES` | |---|---|---| | arrproxy | identity.unkin.net | yes | | logviewer | identity.unkin.net | yes | | mediamark | identity.unkin.net | yes | | repospawner | identity.unkin.net | yes | | watchstate | identity.k8s… | yes | | **artifactapi** | identity.unkin.net | **no** | repospawner uses the **same public `identity.unkin.net` issuer** and still needs the internal bundle, which falsifies the removal reasoning. The existing comment on that initContainer states it plainly: *"The Authentik issuer is served behind the internal unkin.net CA."* ## Changes - Add the `combine-certs` initContainer — byte-identical to repospawner's. - Mount the combined bundle and set `OAUTH2_PROXY_PROVIDER_CA_FILES`. - Reload the Deployment when `vault-ca-cert` rotates. `vault-ca-cert` already exists in the `artifactapi` namespace (`api-deployment.yaml` uses it). `kustomize build apps/base/artifactapi` succeeds. ## Risk Trust-only and strictly additive — it appends the internal CA to the system roots. Harmless if the back channel turns out to reach a publicly trusted endpoint after all. Expected to remove the *next* blocker, surfacing as x509, once the terraform-authentik apply lands. ## Sequencing 1. Fix and re-run terraform-authentik `push/apply` so the `artifactapi` application exists. 2. Merge this. 3. Confirm `/ui/` returns 200, then close #457 unmerged. Only merge #457 instead if the UI must come back before step 1 can be done. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/458 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- .../artifactapi/oauth2-proxy-configmap.yaml | 5 +++ .../artifactapi/oauth2-proxy-deployment.yaml | 45 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/apps/base/artifactapi/oauth2-proxy-configmap.yaml b/apps/base/artifactapi/oauth2-proxy-configmap.yaml index 672b4be..18fbc94 100644 --- a/apps/base/artifactapi/oauth2-proxy-configmap.yaml +++ b/apps/base/artifactapi/oauth2-proxy-configmap.yaml @@ -39,3 +39,8 @@ data: OAUTH2_PROXY_REVERSE_PROXY: "true" OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256" OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true" + # Back-channel discovery/token calls resolve the issuer inside the cluster, + # where it is served under the internal unkin.net CA rather than the publicly + # trusted cert the browser sees. Trust the bundle the combine-certs init + # container assembles, as every other oauth2-proxy in the estate does. + OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt" diff --git a/apps/base/artifactapi/oauth2-proxy-deployment.yaml b/apps/base/artifactapi/oauth2-proxy-deployment.yaml index 765135e..27c7ad4 100644 --- a/apps/base/artifactapi/oauth2-proxy-deployment.yaml +++ b/apps/base/artifactapi/oauth2-proxy-deployment.yaml @@ -6,7 +6,7 @@ metadata: namespace: artifactapi annotations: configmap.reloader.stakater.com/auto: "true" - secret.reloader.stakater.com/reload: "oauth-credentials" + secret.reloader.stakater.com/reload: "oauth-credentials,vault-ca-cert" spec: replicas: 2 selector: @@ -30,6 +30,36 @@ spec: fsGroup: 65532 seccompProfile: type: RuntimeDefault + initContainers: + # The Authentik issuer is served behind the internal unkin.net CA; + # combine the system roots with it so oauth2-proxy's OIDC HTTP client + # trusts the discovery endpoint. + - name: combine-certs + image: docker.io/library/alpine:3 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt + volumeMounts: + - name: vault-ca-cert + mountPath: /custom-ca + readOnly: true + - name: combined-certs + mountPath: /combined-certs + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 64Mi containers: - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3 @@ -83,6 +113,10 @@ spec: capabilities: drop: - ALL + volumeMounts: + - name: combined-certs + mountPath: /etc/ssl/combined + readOnly: true resources: requests: cpu: 50m @@ -90,4 +124,13 @@ spec: limits: cpu: 500m memory: 256Mi + volumes: + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt + - name: combined-certs + emptyDir: {} restartPolicy: Always