From 8ce1b5f6bd76109097c06e14603f84dc6262e13d Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 8 Aug 2026 18:40:37 +1000 Subject: [PATCH] Enable Reloader secret watching, scope existing auto to configmap-only (#326) The internal intermediate CA re-key broke CA consumers because Reloader ignored Secrets and could not restart on the vault-ca-cert Secret. Enabling generic secret watching naively would restart every workload on each Vault/VSO secret rotation, so this scopes existing auto annotations to ConfigMaps and makes secret reload opt-in per Secret. - set reloader ignoreSecrets: false so Secrets are watched - convert every generic reloader.stakater.com/auto to the configmap-only configmap.reloader.stakater.com/auto form (22 annotations, 19 files) - add explicit secret.reloader.stakater.com/reload: "vault-ca-cert" to the CA consumers (artifactapi api, cephrgw-operator, puppetserver master+compiler, litellm, logarchiver) so CA rotation restarts them - add secret.reloader.stakater.com/reload: "kanidm-tls" so the cert-manager leaf renewal rolls kanidm - add docs/ca-rotation.md runbook and index it Closes #326 Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT --- apps/base/age-api/deployment.yaml | 2 +- apps/base/artifactapi/api-deployment.yaml | 3 +- apps/base/artifactapi/ui-deployment.yaml | 2 +- apps/base/cephrgw-system/deployment.yaml | 6 +- apps/base/encapi/deployment.yaml | 2 +- apps/base/kanidm/statefulset.yaml | 3 +- apps/base/litellm/deployment.yaml | 3 +- apps/base/logging/deployment_logarchiver.yaml | 3 +- apps/base/pdbmux/deployment.yaml | 2 +- apps/base/puppet/deployment_puppetboard.yaml | 2 +- apps/base/puppet/deployment_puppetdb.yaml | 2 +- .../deployment_puppetserver-compiler.yaml | 3 +- .../deployment_puppetserver-master.yaml | 5 +- apps/overlays/au-syd1/authentik/values.yaml | 4 +- .../overlays/au-syd1/logging/values-nats.yaml | 5 +- .../au-syd1/logging/values-vector-agent.yaml | 2 +- .../logging/values-vector-aggregator.yaml | 2 +- .../logging/values-vector-vm-ingest.yaml | 2 +- .../au-syd1/reloader-system/values.yaml | 5 +- apps/overlays/au-syd1/woodpecker/values.yaml | 4 +- docs/README.md | 1 + docs/ca-rotation.md | 67 +++++++++++++++++++ 22 files changed, 105 insertions(+), 25 deletions(-) create mode 100644 docs/ca-rotation.md diff --git a/apps/base/age-api/deployment.yaml b/apps/base/age-api/deployment.yaml index f555471..5209092 100644 --- a/apps/base/age-api/deployment.yaml +++ b/apps/base/age-api/deployment.yaml @@ -12,7 +12,7 @@ spec: template: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" labels: app: age-api spec: diff --git a/apps/base/artifactapi/api-deployment.yaml b/apps/base/artifactapi/api-deployment.yaml index 1b27ccc..89dbec7 100644 --- a/apps/base/artifactapi/api-deployment.yaml +++ b/apps/base/artifactapi/api-deployment.yaml @@ -5,7 +5,8 @@ metadata: name: api namespace: artifactapi annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" spec: selector: matchLabels: diff --git a/apps/base/artifactapi/ui-deployment.yaml b/apps/base/artifactapi/ui-deployment.yaml index c274e58..83c2cc2 100644 --- a/apps/base/artifactapi/ui-deployment.yaml +++ b/apps/base/artifactapi/ui-deployment.yaml @@ -5,7 +5,7 @@ metadata: name: ui namespace: artifactapi annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" spec: selector: matchLabels: diff --git a/apps/base/cephrgw-system/deployment.yaml b/apps/base/cephrgw-system/deployment.yaml index d6d5f9e..79e31cf 100644 --- a/apps/base/cephrgw-system/deployment.yaml +++ b/apps/base/cephrgw-system/deployment.yaml @@ -7,8 +7,10 @@ metadata: labels: app.kubernetes.io/name: cephrgw-operator annotations: - # Restart the operator when the credentials Secret rotates. - reloader.stakater.com/auto: "true" + # Restart on internal CA rotation only; cephrgw-credentials is Vault-rotated + # (VSO) and deliberately excluded so routine key rotation causes no restart. + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" spec: replicas: 1 selector: diff --git a/apps/base/encapi/deployment.yaml b/apps/base/encapi/deployment.yaml index 0872a87..aab1a1b 100644 --- a/apps/base/encapi/deployment.yaml +++ b/apps/base/encapi/deployment.yaml @@ -5,7 +5,7 @@ metadata: name: encapi namespace: encapi annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" spec: replicas: 2 selector: diff --git a/apps/base/kanidm/statefulset.yaml b/apps/base/kanidm/statefulset.yaml index cc42fb6..82017ba 100644 --- a/apps/base/kanidm/statefulset.yaml +++ b/apps/base/kanidm/statefulset.yaml @@ -5,7 +5,8 @@ metadata: name: kanidm namespace: kanidm annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "kanidm-tls" labels: app.kubernetes.io/name: kanidm app.kubernetes.io/instance: kanidm diff --git a/apps/base/litellm/deployment.yaml b/apps/base/litellm/deployment.yaml index 0a70f65..2d74a56 100644 --- a/apps/base/litellm/deployment.yaml +++ b/apps/base/litellm/deployment.yaml @@ -11,7 +11,8 @@ spec: template: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" labels: app: litellm spec: diff --git a/apps/base/logging/deployment_logarchiver.yaml b/apps/base/logging/deployment_logarchiver.yaml index d738db7..2a12f78 100644 --- a/apps/base/logging/deployment_logarchiver.yaml +++ b/apps/base/logging/deployment_logarchiver.yaml @@ -10,7 +10,8 @@ metadata: name: logarchiver namespace: logging annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" labels: app.kubernetes.io/name: logarchiver app.kubernetes.io/component: archiver diff --git a/apps/base/pdbmux/deployment.yaml b/apps/base/pdbmux/deployment.yaml index 8f32f78..ae1f23d 100644 --- a/apps/base/pdbmux/deployment.yaml +++ b/apps/base/pdbmux/deployment.yaml @@ -5,7 +5,7 @@ metadata: name: pdbmux namespace: pdbmux annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" spec: replicas: 2 selector: diff --git a/apps/base/puppet/deployment_puppetboard.yaml b/apps/base/puppet/deployment_puppetboard.yaml index 19664a5..97a4b01 100644 --- a/apps/base/puppet/deployment_puppetboard.yaml +++ b/apps/base/puppet/deployment_puppetboard.yaml @@ -21,7 +21,7 @@ spec: template: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" labels: app.kubernetes.io/component: puppetboard app.kubernetes.io/instance: puppetserver diff --git a/apps/base/puppet/deployment_puppetdb.yaml b/apps/base/puppet/deployment_puppetdb.yaml index 14ddddf..2c08999 100644 --- a/apps/base/puppet/deployment_puppetdb.yaml +++ b/apps/base/puppet/deployment_puppetdb.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" labels: app.kubernetes.io/component: puppetdb app.kubernetes.io/instance: puppetserver diff --git a/apps/base/puppet/deployment_puppetserver-compiler.yaml b/apps/base/puppet/deployment_puppetserver-compiler.yaml index af67c89..a21b41d 100644 --- a/apps/base/puppet/deployment_puppetserver-compiler.yaml +++ b/apps/base/puppet/deployment_puppetserver-compiler.yaml @@ -2,7 +2,8 @@ apiVersion: apps/v1 kind: Deployment metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" labels: app.kubernetes.io/component: puppetserver-compilers app.kubernetes.io/instance: puppetserver diff --git a/apps/base/puppet/deployment_puppetserver-master.yaml b/apps/base/puppet/deployment_puppetserver-master.yaml index 9d94c29..54aa4e3 100644 --- a/apps/base/puppet/deployment_puppetserver-master.yaml +++ b/apps/base/puppet/deployment_puppetserver-master.yaml @@ -2,7 +2,8 @@ apiVersion: apps/v1 kind: Deployment metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" + secret.reloader.stakater.com/reload: "vault-ca-cert" labels: app.kubernetes.io/component: puppetserver app.kubernetes.io/instance: puppetserver @@ -20,7 +21,7 @@ spec: template: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" labels: app.kubernetes.io/component: puppetserver app.kubernetes.io/instance: puppetserver diff --git a/apps/overlays/au-syd1/authentik/values.yaml b/apps/overlays/au-syd1/authentik/values.yaml index 067c12e..d2710e9 100644 --- a/apps/overlays/au-syd1/authentik/values.yaml +++ b/apps/overlays/au-syd1/authentik/values.yaml @@ -75,7 +75,7 @@ global: server: replicas: 3 annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" ingress: enabled: false resources: @@ -89,7 +89,7 @@ server: worker: replicas: 2 annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" resources: limits: cpu: "2" diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index fc16bd2..f482dff 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -86,12 +86,13 @@ container: cpu: "2" memory: 4Gi -# Roll the StatefulSet when nats-auth changes. +# Roll the StatefulSet on config changes only; nats-auth is Vault-rotated (VSO) +# so it is deliberately not watched here (no restart on routine key rotation). podTemplate: merge: metadata: annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" # Config-reloader sidecar image, also through artifactapi. reloader: diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml index 1be370d..d20092a 100644 --- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -50,4 +50,4 @@ existingConfigMaps: - vector-agent-config workloadResourceAnnotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml index b4d48f9..f6226b4 100644 --- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -26,7 +26,7 @@ autoscaling: targetCPUUtilizationPercentage: 70 workloadResourceAnnotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" podLabels: vector.dev/exclude: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml index dcb4942..c915c7a 100644 --- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml @@ -10,7 +10,7 @@ image: tag: 0.57.0-distroless-libc workloadResourceAnnotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" podLabels: vector.dev/exclude: "true" diff --git a/apps/overlays/au-syd1/reloader-system/values.yaml b/apps/overlays/au-syd1/reloader-system/values.yaml index 2a2cbf7..a1e53dc 100644 --- a/apps/overlays/au-syd1/reloader-system/values.yaml +++ b/apps/overlays/au-syd1/reloader-system/values.yaml @@ -2,7 +2,10 @@ reloader: autoReloadAll: false isArgoRollouts: true - ignoreSecrets: true + # Watch Secrets so opt-in secret.reloader.stakater.com/reload works (issue + # #326). Workloads use configmap-only auto by default; Vault/VSO-rotated + # Secrets do NOT trigger restarts unless explicitly named. + ignoreSecrets: false ignoreConfigMaps: false ignoreJobs: true ignoreCronJobs: true diff --git a/apps/overlays/au-syd1/woodpecker/values.yaml b/apps/overlays/au-syd1/woodpecker/values.yaml index 57dc0da..a448dfb 100644 --- a/apps/overlays/au-syd1/woodpecker/values.yaml +++ b/apps/overlays/au-syd1/woodpecker/values.yaml @@ -17,7 +17,7 @@ agent: persistence: storageClass: cephrbd-fast-delete podAnnotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" resources: requests: cpu: 50m @@ -40,7 +40,7 @@ server: persistentVolume: storageClass: cephrbd-fast-delete podAnnotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/auto: "true" ingress: enabled: true annotations: diff --git a/docs/README.md b/docs/README.md index 1e790f4..1723eb7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,3 +8,4 @@ Operational notes for the manifests in this repo. | [cnpg-restore.md](cnpg-restore.md) | Restoring a CNPG cluster: full recovery, point-in-time recovery, cutover, and gotchas. | | [authentik-rancher-sso.md](authentik-rancher-sso.md) | Manual runtime step to point Rancher's OIDC auth at the canonical `identity.unkin.net` issuer and trust the internal CA. | | [gitea-migration.md](gitea-migration.md) | Staged cutover of the git.unkin.net forge from the Puppet VM to the `gitea` namespace. | +| [ca-rotation.md](ca-rotation.md) | Rolling the internal `unkin.net` PKI CA (`vault-ca-cert`): what Reloader restarts automatically vs. manual/CNPG restarts. | diff --git a/docs/ca-rotation.md b/docs/ca-rotation.md new file mode 100644 index 0000000..450cecb --- /dev/null +++ b/docs/ca-rotation.md @@ -0,0 +1,67 @@ +# Internal CA rotation runbook + +How to roll the internal `unkin.net` PKI when the intermediate CA is re-keyed, +without turning it into an incident (issue #326). + +Trust in the internal CA is bootstrapped by the plain `vault-ca-cert` Secret in +the `certificates` namespace (it cannot be Vault-sourced — it is what teaches +workloads to trust Vault in the first place). It is replicated into every +namespace by the emberstack reflector. Everything below keys off that Secret. + +## Before you start + +- `vault-ca-cert` MUST carry the **full current chain** (intermediate + root), + not a bare leaf. radosgw (Ceph RGW) serves a bare leaf, so CNPG->RGW backups + and any S3 client verifying against this bundle break if the chain is + incomplete. Update `apps/base/certificates/vault-ca-cert.yaml` with the full + chain and let Argo CD sync it before restarting consumers. + +## What restarts automatically + +Reloader watches Secrets (`ignoreSecrets: false`) but workloads use +`configmap.reloader.stakater.com/auto: "true"` — ConfigMap-only by default — so +Vault/VSO-rotated app Secrets do NOT cause restarts. CA reload is opt-in via an +explicit `secret.reloader.stakater.com/reload: "vault-ca-cert"` annotation. + +These roll on their own when `vault-ca-cert` changes: + +| Workload | Namespace | How it consumes the CA | +| --- | --- | --- | +| `api` (artifactapi) | `artifactapi` | init combines CA into trust bundle | +| `litellm` | `litellm` | init combines CA into trust bundle | +| `cephrgw-operator` | `cephrgw-system` | `CEPH_RGW_CA_FILE` subPath mount | +| `puppetserver-master` | `puppet` | subPath mount | +| `puppetserver-compiler` | `puppet` | subPath mount | +| `logarchiver` | `logging` | subPath mount | + +## What needs a manual restart + +These mount `vault-ca-cert` but are not Reloader-annotated (CRD-managed or +startup-cached), so restart them by hand after the Secret syncs: + +```sh +kubectl -n grafana rollout restart deployment/grafana # grafana-operator Grafana +kubectl -n observability rollout restart deployment/vmagent +kubectl -n paperclip rollout restart deployment/paperclip +kubectl -n argocd rollout restart deployment/argocd-repo-server # startup-cached +``` + +CNPG Postgres clusters cache the CA in the operator; roll each cluster that +trusts `vault-ca-cert` (artifactapi, authentik, encapi, gitea, grafana, +litellm, paperclip, woodpecker): + +```sh +kubectl cnpg restart -n +``` + +## Verify + +Confirm the in-pod bundle matches the new CA: + +```sh +kubectl -n exec deploy/ -- \ + sha256sum /path/to/ca.crt # compare against the new vault-ca-cert +``` + +For RGW-backed backups, confirm a fresh CNPG base backup completes after the +roll.