From 28d0a6ed79103a433f32244ff162d9fd354660e7 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 11 Aug 2026 21:52:21 +1000 Subject: [PATCH] Decouple netbox user management from the rotating engine seed The netbox secrets engine seeds its admin_token from KV once and then rotates it in place (netbox/config/rotate), which mints a fresh admin token and deletes the old one. netbox_user_management re-reads that same KV admin_token on every apply to drive the e-breuninger provider, so after the first rotation it authenticates with a token NetBox has already deleted and the apply fails. A separate failure mode compounds this: a KV admin token carrying a literal "Bearer "/"Token " scheme prefix produces a malformed three-part Authorization header and a 403, because NetBox derives the token version from the value's nbt_ prefix (not the keyword) and expects a bare token. - Read a dedicated, never-rotated user_mgmt_token for user management, falling back to admin_token only as a bootstrap convenience until it is seeded. - Add a check block that warns when only the rotating seed is present, nudging operators to seed user_mgmt_token before enabling engine rotation. - Add data-source postconditions on both modules rejecting a scheme-prefixed admin credential with a clear message instead of a downstream 403. - Document the two KV keys and the bare-token contract in the backend config. --- config/netbox_secret_backend/netbox.yaml | 17 +++++- .../modules/netbox_secret_backend/main.tf | 13 +++++ .../modules/netbox_user_management/main.tf | 53 +++++++++++++++++-- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/config/netbox_secret_backend/netbox.yaml b/config/netbox_secret_backend/netbox.yaml index 4e22f42..cc877e4 100644 --- a/config/netbox_secret_backend/netbox.yaml +++ b/config/netbox_secret_backend/netbox.yaml @@ -1,12 +1,25 @@ # Mounts the netbox token secrets engine at "netbox" and writes its config. # The seeded NetBox admin token is sensitive and read from KV, not stored here: # kv/service/vault/au/syd1/secret_backend/netbox/config -# -> key: admin_token (required) -# Populate that KV path with a purpose-built NetBox service token that has +# -> key: admin_token (required) engine bootstrap seed; rotated by Vault +# -> key: user_mgmt_token (recommended) stable token for netbox_user_management +# +# Both keys must be BARE NetBox tokens with NO scheme prefix: do not prepend +# "Bearer " or "Token ". NetBox infers the version from the value's nbt_ prefix, +# so one bare token authenticates under either scheme; the plugin and provider +# add the keyword themselves. A prefixed value yields a malformed header + 403. +# +# Populate admin_token with a purpose-built NetBox service token that has # add_token + grant_token (or superuser) BEFORE applying, then run # `vault write -f netbox/config/rotate` after the first apply so only Vault # holds the live admin token. # +# Rotation caveat: config/rotate mints a fresh admin_token and DELETES the old +# one, so the KV admin_token becomes a dead token. netbox_user_management reads +# its credential every apply, so seed a SEPARATE, never-rotated user_mgmt_token +# (also add_user + add_token + grant_token, or superuser) BEFORE rotating; +# otherwise user management breaks once admin_token is rotated away. +# # token_version 2 is the NetBox 4.6.5 default and requires API_TOKEN_PEPPERS to # be configured on the NetBox server; set token_version: 1 here if the server # has no peppers. diff --git a/modules/vault_cluster/modules/netbox_secret_backend/main.tf b/modules/vault_cluster/modules/netbox_secret_backend/main.tf index 19c99b0..48dbfb5 100644 --- a/modules/vault_cluster/modules/netbox_secret_backend/main.tf +++ b/modules/vault_cluster/modules/netbox_secret_backend/main.tf @@ -8,6 +8,19 @@ data "vault_kv_secret_v2" "config" { mount = "kv" name = "service/vault/${var.country}/${var.region}/secret_backend/${var.path}/config" + + lifecycle { + # The plugin adds the Authorization scheme itself (Bearer for v2 nbt_ tokens, + # Token for v1), so admin_token must be the BARE token. A literal `Bearer `/ + # `Token ` prefix in the seed yields a malformed header and 403s on mint/rotate. + postcondition { + condition = nonsensitive( + !startswith(self.data["admin_token"], "Bearer ") && + !startswith(self.data["admin_token"], "Token ") + ) + error_message = "KV admin_token for netbox backend '${var.path}' must be a BARE NetBox token with no 'Bearer '/'Token ' scheme prefix (v2: nbt_.; v1: the 40-char value)." + } + } } resource "netbox_secret_backend" "this" { diff --git a/modules/vault_cluster/modules/netbox_user_management/main.tf b/modules/vault_cluster/modules/netbox_user_management/main.tf index a5692d0..a0d0d1c 100644 --- a/modules/vault_cluster/modules/netbox_user_management/main.tf +++ b/modules/vault_cluster/modules/netbox_user_management/main.tf @@ -1,11 +1,56 @@ -# Read the seeded NetBox admin token for each backend from KV. This is the same -# token the netbox secrets engine is configured with (key admin_token), and it -# must carry add_token + grant_token (or superuser) to create users/permissions. +# Read the NetBox admin credential for each backend from KV. User management runs +# on EVERY apply (the e-breuninger provider refreshes/reconciles the users and +# permissions), so it needs a STABLE admin token, not the engine's bootstrap seed. +# +# The netbox secrets engine seeds its `admin_token` once and then rotates it in +# place (`vault write -f netbox/config/rotate`), which mints a fresh admin token +# and DELETES the old one - so after the first rotation the KV `admin_token` names +# a token that no longer exists in NetBox. Binding user management to it would make +# every post-rotation apply fail to authenticate. We therefore prefer a dedicated, +# never-rotated key `user_mgmt_token` and fall back to `admin_token` only as a +# bootstrap convenience (valid until the first rotate). Seed `user_mgmt_token` +# (bare token, add_user + add_token + grant_token, or superuser) BEFORE rotating. data "vault_kv_secret_v2" "netbox_backend_configs" { for_each = var.netbox_backends mount = "kv" name = "service/vault/${var.country}/${var.region}/secret_backend/${each.key}/config" + + lifecycle { + # NetBox derives the token version from the value's `nbt_` prefix, not the + # Authorization keyword, so the same BARE token works whether a client sends + # it as `Token ` (e-breuninger provider) or `Bearer ` (the plugin). A + # value carrying a literal `Bearer `/`Token ` scheme prefix produces a + # malformed three-part header and a 403; reject it here with a clear message. + postcondition { + condition = nonsensitive( + !startswith(lookup(self.data, "user_mgmt_token", self.data["admin_token"]), "Bearer ") && + !startswith(lookup(self.data, "user_mgmt_token", self.data["admin_token"]), "Token ") + ) + error_message = "KV admin credential for netbox backend '${each.key}' must be a BARE NetBox token with no 'Bearer '/'Token ' scheme prefix (v2: nbt_.; v1: the 40-char value)." + } + } +} + +# Warn (non-fatal) when only the rotating bootstrap seed is present, so operators +# seed a stable user_mgmt_token before enabling engine rotation. +check "netbox_user_mgmt_token_seeded" { + assert { + condition = nonsensitive(alltrue([ + for cfg in data.vault_kv_secret_v2.netbox_backend_configs : + contains(keys(cfg.data), "user_mgmt_token") + ])) + error_message = "A netbox backend has no dedicated user_mgmt_token; user management is falling back to the rotating admin_token seed. Seed a stable user_mgmt_token before running netbox/config/rotate, or user management will fail after the seed is rotated away." + } +} + +locals { + # Prefer the stable, never-rotated user-management token; fall back to the + # bootstrap admin_token only until a user_mgmt_token is seeded. + netbox_admin_tokens = { + for backend, cfg in data.vault_kv_secret_v2.netbox_backend_configs : + backend => lookup(cfg.data, "user_mgmt_token", cfg.data["admin_token"]) + } } # One NetBox provider instance per backend, authenticated with its admin token. @@ -14,7 +59,7 @@ provider "netbox" { for_each = var.netbox_backend_aliases server_url = var.netbox_backends[each.key].netbox_url - api_token = data.vault_kv_secret_v2.netbox_backend_configs[each.key].data["admin_token"] + api_token = local.netbox_admin_tokens[each.key] allow_insecure_https = var.netbox_backends[each.key].tls_skip_verify # NetBox is internal and not always reachable at plan time; the resource CRUD # calls surface any real incompatibility, so skip the startup version probe.