From a63383555082ba49e88e247f651e380c7ba2ecf3 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 25 Jul 2026 17:39:18 +1000 Subject: [PATCH] Enable ACLs on k8s consul with Vault-sourced bootstrap token (#280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Phase 1 of the consul VM to k8s migration: bring the k8s consul cluster to ACL parity with the authoritative VM cluster before the eventual snapshot-restore. The VM cluster runs ACLs enabled with `default_policy: deny` and `down_policy: extend-cache`; the k8s cluster currently runs with **ACLs disabled**. Sourcing the bootstrap/management token from Vault lets the k8s cluster bootstrap with the **same** `initial_management` token as the VM cluster, so puppet automation and the snapshot-restore line up. No token material is placed in git. ## Changes - Enable `global.acls.manageSystemACLs` so the chart manages system ACL tokens/policies for consul components. - Point `global.acls.bootstrapToken` at a pre-existing Kubernetes secret `consul-bootstrap-acl-token` (key `token`); chart 1.9.7 supports this, and when the secret is populated the `server-acl-init` job **skips bootstrapping** and adopts that token as the management token (renders `-bootstrap-token-secret-name`/`-bootstrap-token-secret-key`, verified in the kustomize output). - Add a `VaultAuth` (mount `k8s/au/syd1`, role `default`, SA `default`) and `VaultStaticSecret` in the `consul` namespace that sync `kv/kubernetes/namespace/consul/default/bootstrap-acl-token` into the `consul-bootstrap-acl-token` k8s secret via VSO (mirrors the encapi pattern). - Merge the `acl` block (`enabled`, `default_policy: deny`, `down_policy: extend-cache`, `enable_token_persistence`) into the server `extraConfig` to match the VM posture. ## OPERATIONAL NOTE — required BEFORE merge The user MUST place the VM cluster's `initial_management` token in Vault first, or ACL bootstrapping will generate a *different* token and break the mirror: ``` vault kv put kv/kubernetes/namespace/consul/default/bootstrap-acl-token token= ``` VSO then syncs it into the `consul-bootstrap-acl-token` secret before the `server-acl-init` job runs. No terraform-vault change is needed: the wildcard `default` k8s-auth role (`bound_service_account_namespaces: ['*']`) plus the templated `kv/kubernetes/default` policy already grant the `consul` namespace `default` SA read on `kv/kubernetes/namespace/consul/default/*`. ## Risk / expected behavior - Enabling ACLs **rolls the 5 servers** (StatefulSet update) and runs a `server-acl-init` job. - With `default_policy: deny`, previously-anonymous operations are denied. The `vault` service in the k8s catalog is self-registered by the Vault/OpenBao servers (namespace `vault`) via their `service_registration "consul"` stanza (catalog entry has `ServiceMeta.external-source: vault`, port 8200). After the flip this registration will be **denied** unless Vault is given a Consul ACL token with `service:write` on `vault` (and the anonymous token is not granted that). This must be handled as part of the cutover — either grant the anonymous token limited write, or configure a token in Vault's consul service_registration. - Anonymous HTTP API reads (e.g. `/v1/status/leader`) will also be denied post-merge unless a token is supplied — see PR 2 verification note. ## Ordering Independent of the API-route PR (`benvin/consul-api-route`). Both precede phase 3 (snapshot). Do not merge until the Vault step above is done. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/280 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- apps/base/consul/kustomization.yaml | 2 ++ apps/base/consul/vaultauth.yaml | 18 ++++++++++++++++++ apps/base/consul/vaultstaticsecret.yaml | 17 +++++++++++++++++ apps/overlays/au-syd1/consul/values.yaml | 19 +++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 apps/base/consul/vaultauth.yaml create mode 100644 apps/base/consul/vaultstaticsecret.yaml diff --git a/apps/base/consul/kustomization.yaml b/apps/base/consul/kustomization.yaml index 2c2b5da..217d383 100644 --- a/apps/base/consul/kustomization.yaml +++ b/apps/base/consul/kustomization.yaml @@ -6,3 +6,5 @@ resources: - namespace.yaml - gateway.yaml - httproute.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml diff --git a/apps/base/consul/vaultauth.yaml b/apps/base/consul/vaultauth.yaml new file mode 100644 index 0000000..af9bee0 --- /dev/null +++ b/apps/base/consul/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: consul +spec: + allowedNamespaces: + - consul + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/consul/vaultstaticsecret.yaml b/apps/base/consul/vaultstaticsecret.yaml new file mode 100644 index 0000000..6cd7a6d --- /dev/null +++ b/apps/base/consul/vaultstaticsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: bootstrap-acl-token + namespace: consul +spec: + destination: + create: true + name: consul-bootstrap-acl-token + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/consul/default/bootstrap-acl-token + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/overlays/au-syd1/consul/values.yaml b/apps/overlays/au-syd1/consul/values.yaml index 424c6ee..cc9f434 100644 --- a/apps/overlays/au-syd1/consul/values.yaml +++ b/apps/overlays/au-syd1/consul/values.yaml @@ -3,6 +3,19 @@ global: datacenter: au-syd1 domain: consul + acls: + # Enable chart-managed ACL tokens/policies for Consul system components. + manageSystemACLs: true + # Source the bootstrap/management token from a pre-existing Kubernetes secret + # instead of letting the chart generate one. The secret is synced from Vault + # via VSO (see ../../../base/consul/vaultauth.yaml and vaultstaticsecret.yaml). + # When this secret is populated the server-acl-init job SKIPS bootstrapping and + # uses the supplied token as the management token, so the k8s cluster bootstraps + # with the SAME initial_management token as the authoritative VM cluster. + bootstrapToken: + secretName: consul-bootstrap-acl-token + secretKey: token + server: image: hashicorp/consul:1.22.7 replicas: 5 @@ -17,6 +30,12 @@ server: extraConfig: | { + "acl": { + "enabled": true, + "default_policy": "deny", + "down_policy": "extend-cache", + "enable_token_persistence": true + }, "disable_remote_exec": true, "disable_update_check": true, "performance": {