fix: grant vault deployer access to manage the litellm engine (#84)

## Why
Applying the newly-merged litellm mount (#83) failed at apply time with:

```
Error: failed to write litellm config
URL: PUT https://vault.service.consul:8200/v1/litellm/config
Code: 403. * permission denied
```

The deployer identity (`tf_vault` approle / `woodpecker_terraform_vault` k8s role) can enable the mount via `sys/mounts/admin`, but no policy grants it access to the engine's own data paths, so writing the config and roles is denied.

## Changes
- Add `policies/litellm/admin.yaml` granting `create`/`read`/`update`/`delete` on `litellm/config` and `litellm/roles/*` (plus `read`/`list` on `litellm/roles`), assigned to the same auth roles as the other secret-engine admin policies (`tf_vault`, `woodpecker_terraform_vault`).

## Note
The policy attaches to the deployer's auth roles, so it takes effect on the next token issuance — a re-run of the apply (fresh Vault login) will have the permission and can write `litellm/config` and the roles.

Reviewed-on: #84
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>

config/auth_approle_role/approle/terraform_artifactapi.yaml

@@ -0,0 +1,9 @@
+token_ttl: 120
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: true

config/auth_approle_role/approle/terraform_authentik.yaml

@@ -0,0 +1,9 @@
+token_ttl: 120
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: true

config/auth_approle_role/approle/terraform_prowlarr.yaml

@@ -0,0 +1,9 @@
+token_ttl: 120
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: true

config/auth_approle_role/approle/terraform_radarr.yaml

@@ -0,0 +1,9 @@
+token_ttl: 120
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: true

config/auth_approle_role/approle/terraform_sonarr.yaml

@@ -0,0 +1,9 @@
+token_ttl: 120
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: true

config/auth_kubernetes_role/k8s/au/syd1/woodpecker_terraform_artifactapi.yaml

@@ -0,0 +1,7 @@
+bound_service_account_names:
+  - terraform-artifactapi
+bound_service_account_namespaces:
+  - woodpecker
+token_ttl: 600
+token_max_ttl: 600
+audience: https://kubernetes.default.svc.cluster.local

config/auth_kubernetes_role/k8s/au/syd1/woodpecker_terraform_authentik.yaml

@@ -0,0 +1,7 @@
+bound_service_account_names:
+  - terraform-authentik
+bound_service_account_namespaces:
+  - woodpecker
+token_ttl: 600
+token_max_ttl: 600
+audience: https://kubernetes.default.svc.cluster.local

config/auth_kubernetes_role/k8s/au/syd1/woodpecker_terraform_prowlarr.yaml

@@ -0,0 +1,7 @@
+bound_service_account_names:
+  - terraform-prowlarr
+bound_service_account_namespaces:
+  - woodpecker
+token_ttl: 600
+token_max_ttl: 600
+audience: https://kubernetes.default.svc.cluster.local

config/auth_kubernetes_role/k8s/au/syd1/woodpecker_terraform_radarr.yaml

@@ -1,8 +1,7 @@
 bound_service_account_names:
-  - default
-  - forgebot-operator
+  - terraform-radarr
 bound_service_account_namespaces:
-  - forgebot
+  - woodpecker
 token_ttl: 600
 token_max_ttl: 600
-audience: vault
+audience: https://kubernetes.default.svc.cluster.local

config/auth_kubernetes_role/k8s/au/syd1/woodpecker_terraform_sonarr.yaml

@@ -0,0 +1,7 @@
+bound_service_account_names:
+  - terraform-sonarr
+bound_service_account_namespaces:
+  - woodpecker
+token_ttl: 600
+token_max_ttl: 600
+audience: https://kubernetes.default.svc.cluster.local

config/config.hcl

@@ -185,5 +185,18 @@ locals {
       trimsuffix(basename(file_path), ".yaml") => content
       if startswith(file_path, "pki_mount_only/")
     }
+    litellm_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "litellm_secret_backend/")
+    }
+    litellm_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "litellm_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "litellm_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "litellm_secret_backend_role/")
+    }
   }
 }

config/consul_secret_backend_role/consul_root/au/syd1/terraform-artifactapi.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-artifactapi
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-authentik.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-authentik
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-prowlarr.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-prowlarr
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-radarr.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-radarr
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-sonarr.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-sonarr
+ttl: 120
+max_ttl: 300
+datacenters: []

config/litellm_secret_backend/litellm.yaml

@@ -0,0 +1,6 @@
+# Mounts the LiteLLM dynamic secrets engine at "litellm" and writes its config.
+# The master key is sensitive and read from KV, not stored here:
+#   kv/service/vault/au/syd1/secret_backend/litellm -> key "master_key"
+description: "LiteLLM dynamic virtual keys"
+base_url: "http://litellm.litellm.svc:4000"
+request_timeout_seconds: 30

config/litellm_secret_backend_role/litellm/rspamd.yaml

@@ -0,0 +1,10 @@
+---
+models:
+  - claude-haiku-4-5
+  - claude-sonnet-4-6
+max_budget: 50
+ttl: 3600      # seconds (1h)
+max_ttl: 86400 # seconds (24h)
+metadata:
+  team: mailfiltering
+  env: prod

environments/au/syd1/terragrunt.hcl

@@ -68,6 +68,8 @@ inputs = {
   kubernetes_secret_backend      = local.config.kubernetes_secret_backend
   kubernetes_secret_backend_role = local.config.kubernetes_secret_backend_role
   pki_mount_only                 = local.config.pki_mount_only
+  litellm_secret_backend         = local.config.litellm_secret_backend
+  litellm_secret_backend_role    = local.config.litellm_secret_backend_role
 
   # Pass policy maps to vault_cluster module
   policy_auth_map  = local.policies.policy_auth_map

environments/root.hcl

@@ -11,6 +11,12 @@ provider "vault" {
   address = local.vault_addr
 }
 
+# The LiteLLM secrets engine is managed through its own provider, which talks to
+# the same Vault server. Token falls back to the VAULT_TOKEN environment variable.
+provider "litellm" {
+  address = local.vault_addr
+}
+
 terraform {
   backend "consul" {
     address = "https://consul.service.consul"
@@ -29,6 +35,10 @@ terraform {
       source  = "hashicorp/consul"
       version = "2.23.0"
     }
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
   }
 }
 EOF

modules/vault_cluster/main.tf

@@ -303,6 +303,37 @@ module "kubernetes_secret_backend_role" {
   depends_on = [module.kubernetes_secret_backend]
 }
 
+module "litellm_secret_backend" {
+  source = "./modules/litellm_secret_backend"
+
+  for_each = var.litellm_secret_backend
+
+  country                 = var.country
+  region                  = var.region
+  path                    = each.key
+  plugin                  = each.value.plugin
+  description             = each.value.description
+  base_url                = each.value.base_url
+  request_timeout_seconds = each.value.request_timeout_seconds
+}
+
+module "litellm_secret_backend_role" {
+  source = "./modules/litellm_secret_backend_role"
+
+  for_each = var.litellm_secret_backend_role
+
+  name             = each.value.name
+  backend          = each.value.backend
+  models           = each.value.models
+  max_budget       = each.value.max_budget
+  key_alias_prefix = each.value.key_alias_prefix
+  ttl              = each.value.ttl
+  max_ttl          = each.value.max_ttl
+  metadata         = each.value.metadata
+
+  depends_on = [module.litellm_secret_backend]
+}
+
 module "vault_policy" {
   source = "./modules/vault_policy"
 

modules/vault_cluster/modules/litellm_secret_backend/main.tf

@@ -0,0 +1,14 @@
+# Expected keys in KV secret: master_key
+data "vault_kv_secret_v2" "secret_backend_config" {
+  mount = "kv"
+  name  = "service/vault/${var.country}/${var.region}/secret_backend/${var.path}"
+}
+
+resource "litellm_secret_backend" "this" {
+  path                    = var.path
+  plugin                  = var.plugin
+  description             = var.description
+  base_url                = var.base_url
+  master_key              = data.vault_kv_secret_v2.secret_backend_config.data["master_key"]
+  request_timeout_seconds = var.request_timeout_seconds
+}

modules/vault_cluster/modules/litellm_secret_backend/terraform.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 1.10"
+  required_providers {
+    vault = {
+      source  = "hashicorp/vault"
+      version = "5.6.0"
+    }
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
+  }
+}

modules/vault_cluster/modules/litellm_secret_backend/variables.tf

@@ -0,0 +1,37 @@
+variable "country" {
+  description = "Country identifier (used to locate the KV secret holding the master key)"
+  type        = string
+}
+
+variable "region" {
+  description = "Region identifier (used to locate the KV secret holding the master key)"
+  type        = string
+}
+
+variable "path" {
+  description = "Mount path of the LiteLLM secrets engine"
+  type        = string
+}
+
+variable "plugin" {
+  description = "Registered plugin name/type to mount"
+  type        = string
+  default     = "vault-plugin-secrets-litellm"
+}
+
+variable "description" {
+  description = "Human-friendly description of the mount"
+  type        = string
+  default     = null
+}
+
+variable "base_url" {
+  description = "Base URL of the LiteLLM proxy (e.g. http://litellm.litellm.svc:4000)"
+  type        = string
+}
+
+variable "request_timeout_seconds" {
+  description = "HTTP timeout in seconds for calls from the plugin to the LiteLLM proxy"
+  type        = number
+  default     = 30
+}

modules/vault_cluster/modules/litellm_secret_backend_role/main.tf

@@ -0,0 +1,10 @@
+resource "litellm_secret_backend_role" "this" {
+  backend          = var.backend
+  name             = var.name
+  models           = var.models
+  max_budget       = var.max_budget
+  key_alias_prefix = var.key_alias_prefix
+  ttl              = var.ttl
+  max_ttl          = var.max_ttl
+  metadata         = var.metadata
+}

modules/vault_cluster/modules/litellm_secret_backend_role/terraform.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 1.10"
+  required_providers {
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
+  }
+}

modules/vault_cluster/modules/litellm_secret_backend_role/variables.tf

@@ -0,0 +1,45 @@
+variable "name" {
+  description = "Name of the role"
+  type        = string
+}
+
+variable "backend" {
+  description = "Mount path of the LiteLLM secrets engine this role belongs to"
+  type        = string
+}
+
+variable "models" {
+  description = "Models a generated key may access. Empty means unrestricted"
+  type        = list(string)
+  default     = null
+}
+
+variable "max_budget" {
+  description = "Spending limit applied to each generated key. 0 means unlimited"
+  type        = number
+  default     = null
+}
+
+variable "key_alias_prefix" {
+  description = "Prefix for the auto-generated key alias"
+  type        = string
+  default     = null
+}
+
+variable "ttl" {
+  description = "Default lease TTL in seconds for keys generated from this role"
+  type        = number
+  default     = null
+}
+
+variable "max_ttl" {
+  description = "Maximum lease TTL in seconds for keys generated from this role"
+  type        = number
+  default     = null
+}
+
+variable "metadata" {
+  description = "Metadata attached to each generated key"
+  type        = map(string)
+  default     = null
+}

modules/vault_cluster/variables.tf

@@ -289,6 +289,32 @@ variable "kubernetes_secret_backend_role" {
   default = {}
 }
 
+variable "litellm_secret_backend" {
+  description = "Map of LiteLLM secret engines to create (mount + config). The master key is read from KV"
+  type = map(object({
+    plugin                  = optional(string, "vault-plugin-secrets-litellm")
+    description             = optional(string)
+    base_url                = string
+    request_timeout_seconds = optional(number, 30)
+  }))
+  default = {}
+}
+
+variable "litellm_secret_backend_role" {
+  description = "Map of LiteLLM roles to create"
+  type = map(object({
+    name             = string
+    backend          = string
+    models           = optional(list(string))
+    max_budget       = optional(number)
+    key_alias_prefix = optional(string)
+    ttl              = optional(number)
+    max_ttl          = optional(number)
+    metadata         = optional(map(string))
+  }))
+  default = {}
+}
+
 variable "policy_auth_map" {
   description = "Map of auth mounts -> auth roles -> policy names"
   type        = map(map(list(string)))

policies/consul_root/au/syd1/creds/terraform-artifactapi.yaml

@@ -0,0 +1,11 @@
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-artifactapi"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_artifactapi
+  k8s/au/syd1:
+    - woodpecker_terraform_artifactapi

policies/consul_root/au/syd1/creds/terraform-authentik.yaml

@@ -0,0 +1,11 @@
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-authentik"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_authentik
+  k8s/au/syd1:
+    - woodpecker_terraform_authentik

policies/consul_root/au/syd1/creds/terraform-prowlarr.yaml

@@ -0,0 +1,11 @@
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-prowlarr"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_prowlarr
+  k8s/au/syd1:
+    - woodpecker_terraform_prowlarr

policies/consul_root/au/syd1/creds/terraform-radarr.yaml

@@ -0,0 +1,11 @@
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-radarr"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_radarr
+  k8s/au/syd1:
+    - woodpecker_terraform_radarr

policies/consul_root/au/syd1/creds/terraform-sonarr.yaml

@@ -0,0 +1,11 @@
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-sonarr"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_sonarr
+  k8s/au/syd1:
+    - woodpecker_terraform_sonarr

policies/kv/service/media-apps/prowlarr/read.yaml

@@ -6,5 +6,8 @@ rules:
       - read
 
 auth:
+  approle:
+    - terraform_prowlarr
   k8s/au/syd1:
     - media-apps
+    - woodpecker_terraform_prowlarr

policies/kv/service/media-apps/radarr/read.yaml

@@ -6,5 +6,8 @@ rules:
       - read
 
 auth:
+  approle:
+    - terraform_radarr
   k8s/au/syd1:
     - media-apps
+    - woodpecker_terraform_radarr

policies/kv/service/media-apps/sonarr/read.yaml

@@ -6,5 +6,8 @@ rules:
       - read
 
 auth:
+  approle:
+    - terraform_sonarr
   k8s/au/syd1:
     - media-apps
+    - woodpecker_terraform_sonarr

policies/kv/service/terraform/authentik.yaml

@@ -0,0 +1,18 @@
+# Allow the Terraform Authentik runner to read:
+#  - its own Authentik API token (provider auth), and
+#  - OAuth2/OIDC client secrets that back authentik providers (per target
+#    service's kv namespace path, via the module's data.vault_kv_secret_v2).
+---
+rules:
+  - path: "kv/data/service/terraform/authentik"
+    capabilities:
+      - read
+  - path: "kv/data/kubernetes/namespace/+/default/oauth-credentials"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_authentik
+  k8s/au/syd1:
+    - woodpecker_terraform_authentik

policies/litellm/admin.yaml

@@ -0,0 +1,26 @@
+# Allow management of the LiteLLM secrets engine (config and roles)
+---
+rules:
+  - path: "litellm/config"
+    capabilities:
+      - create
+      - update
+      - read
+      - delete
+  - path: "litellm/roles/*"
+    capabilities:
+      - create
+      - update
+      - delete
+      - read
+      - list
+  - path: "litellm/roles"
+    capabilities:
+      - read
+      - list
+
+auth:
+  approle:
+    - tf_vault
+  k8s/au/syd1:
+    - woodpecker_terraform_vault

resources/secret_backend/consul_root/au/syd1/terraform-artifactapi.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/artifactapi/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-authentik.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/authentik/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-prowlarr.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/prowlarr/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-radarr.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/radarr/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-sonarr.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/sonarr/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}