fix: grant vault deployer access to manage the litellm engine (#84)

## Why
Applying the newly-merged litellm mount (#83) failed at apply time with:

```
Error: failed to write litellm config
URL: PUT https://vault.service.consul:8200/v1/litellm/config
Code: 403. * permission denied
```

The deployer identity (`tf_vault` approle / `woodpecker_terraform_vault` k8s role) can enable the mount via `sys/mounts/admin`, but no policy grants it access to the engine's own data paths, so writing the config and roles is denied.

## Changes
- Add `policies/litellm/admin.yaml` granting `create`/`read`/`update`/`delete` on `litellm/config` and `litellm/roles/*` (plus `read`/`list` on `litellm/roles`), assigned to the same auth roles as the other secret-engine admin policies (`tf_vault`, `woodpecker_terraform_vault`).

## Note
The policy attaches to the deployer's auth roles, so it takes effect on the next token issuance — a re-run of the apply (fresh Vault login) will have the permission and can write `litellm/config` and the roles.

Reviewed-on: #84
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>

config/config.hcl

@@ -185,5 +185,18 @@ locals {
       trimsuffix(basename(file_path), ".yaml") => content
       if startswith(file_path, "pki_mount_only/")
     }
+    litellm_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "litellm_secret_backend/")
+    }
+    litellm_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "litellm_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "litellm_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "litellm_secret_backend_role/")
+    }
   }
 }

config/litellm_secret_backend/litellm.yaml

@@ -0,0 +1,6 @@
+# Mounts the LiteLLM dynamic secrets engine at "litellm" and writes its config.
+# The master key is sensitive and read from KV, not stored here:
+#   kv/service/vault/au/syd1/secret_backend/litellm -> key "master_key"
+description: "LiteLLM dynamic virtual keys"
+base_url: "http://litellm.litellm.svc:4000"
+request_timeout_seconds: 30

config/litellm_secret_backend_role/litellm/rspamd.yaml

@@ -0,0 +1,10 @@
+---
+models:
+  - claude-haiku-4-5
+  - claude-sonnet-4-6
+max_budget: 50
+ttl: 3600      # seconds (1h)
+max_ttl: 86400 # seconds (24h)
+metadata:
+  team: mailfiltering
+  env: prod

environments/au/syd1/terragrunt.hcl

@@ -68,6 +68,8 @@ inputs = {
   kubernetes_secret_backend      = local.config.kubernetes_secret_backend
   kubernetes_secret_backend_role = local.config.kubernetes_secret_backend_role
   pki_mount_only                 = local.config.pki_mount_only
+  litellm_secret_backend         = local.config.litellm_secret_backend
+  litellm_secret_backend_role    = local.config.litellm_secret_backend_role
 
   # Pass policy maps to vault_cluster module
   policy_auth_map  = local.policies.policy_auth_map

environments/root.hcl

@@ -11,6 +11,12 @@ provider "vault" {
   address = local.vault_addr
 }
 
+# The LiteLLM secrets engine is managed through its own provider, which talks to
+# the same Vault server. Token falls back to the VAULT_TOKEN environment variable.
+provider "litellm" {
+  address = local.vault_addr
+}
+
 terraform {
   backend "consul" {
     address = "https://consul.service.consul"
@@ -29,6 +35,10 @@ terraform {
       source  = "hashicorp/consul"
       version = "2.23.0"
     }
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
   }
 }
 EOF

modules/vault_cluster/main.tf

@@ -303,6 +303,37 @@ module "kubernetes_secret_backend_role" {
   depends_on = [module.kubernetes_secret_backend]
 }
 
+module "litellm_secret_backend" {
+  source = "./modules/litellm_secret_backend"
+
+  for_each = var.litellm_secret_backend
+
+  country                 = var.country
+  region                  = var.region
+  path                    = each.key
+  plugin                  = each.value.plugin
+  description             = each.value.description
+  base_url                = each.value.base_url
+  request_timeout_seconds = each.value.request_timeout_seconds
+}
+
+module "litellm_secret_backend_role" {
+  source = "./modules/litellm_secret_backend_role"
+
+  for_each = var.litellm_secret_backend_role
+
+  name             = each.value.name
+  backend          = each.value.backend
+  models           = each.value.models
+  max_budget       = each.value.max_budget
+  key_alias_prefix = each.value.key_alias_prefix
+  ttl              = each.value.ttl
+  max_ttl          = each.value.max_ttl
+  metadata         = each.value.metadata
+
+  depends_on = [module.litellm_secret_backend]
+}
+
 module "vault_policy" {
   source = "./modules/vault_policy"
 

modules/vault_cluster/modules/litellm_secret_backend/main.tf

@@ -0,0 +1,14 @@
+# Expected keys in KV secret: master_key
+data "vault_kv_secret_v2" "secret_backend_config" {
+  mount = "kv"
+  name  = "service/vault/${var.country}/${var.region}/secret_backend/${var.path}"
+}
+
+resource "litellm_secret_backend" "this" {
+  path                    = var.path
+  plugin                  = var.plugin
+  description             = var.description
+  base_url                = var.base_url
+  master_key              = data.vault_kv_secret_v2.secret_backend_config.data["master_key"]
+  request_timeout_seconds = var.request_timeout_seconds
+}

modules/vault_cluster/modules/litellm_secret_backend/terraform.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 1.10"
+  required_providers {
+    vault = {
+      source  = "hashicorp/vault"
+      version = "5.6.0"
+    }
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
+  }
+}

modules/vault_cluster/modules/litellm_secret_backend/variables.tf

@@ -0,0 +1,37 @@
+variable "country" {
+  description = "Country identifier (used to locate the KV secret holding the master key)"
+  type        = string
+}
+
+variable "region" {
+  description = "Region identifier (used to locate the KV secret holding the master key)"
+  type        = string
+}
+
+variable "path" {
+  description = "Mount path of the LiteLLM secrets engine"
+  type        = string
+}
+
+variable "plugin" {
+  description = "Registered plugin name/type to mount"
+  type        = string
+  default     = "vault-plugin-secrets-litellm"
+}
+
+variable "description" {
+  description = "Human-friendly description of the mount"
+  type        = string
+  default     = null
+}
+
+variable "base_url" {
+  description = "Base URL of the LiteLLM proxy (e.g. http://litellm.litellm.svc:4000)"
+  type        = string
+}
+
+variable "request_timeout_seconds" {
+  description = "HTTP timeout in seconds for calls from the plugin to the LiteLLM proxy"
+  type        = number
+  default     = 30
+}

modules/vault_cluster/modules/litellm_secret_backend_role/main.tf

@@ -0,0 +1,10 @@
+resource "litellm_secret_backend_role" "this" {
+  backend          = var.backend
+  name             = var.name
+  models           = var.models
+  max_budget       = var.max_budget
+  key_alias_prefix = var.key_alias_prefix
+  ttl              = var.ttl
+  max_ttl          = var.max_ttl
+  metadata         = var.metadata
+}

modules/vault_cluster/modules/litellm_secret_backend_role/terraform.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 1.10"
+  required_providers {
+    litellm = {
+      source  = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/litellmvaultsecret"
+      version = "0.1.0"
+    }
+  }
+}

modules/vault_cluster/modules/litellm_secret_backend_role/variables.tf

@@ -0,0 +1,45 @@
+variable "name" {
+  description = "Name of the role"
+  type        = string
+}
+
+variable "backend" {
+  description = "Mount path of the LiteLLM secrets engine this role belongs to"
+  type        = string
+}
+
+variable "models" {
+  description = "Models a generated key may access. Empty means unrestricted"
+  type        = list(string)
+  default     = null
+}
+
+variable "max_budget" {
+  description = "Spending limit applied to each generated key. 0 means unlimited"
+  type        = number
+  default     = null
+}
+
+variable "key_alias_prefix" {
+  description = "Prefix for the auto-generated key alias"
+  type        = string
+  default     = null
+}
+
+variable "ttl" {
+  description = "Default lease TTL in seconds for keys generated from this role"
+  type        = number
+  default     = null
+}
+
+variable "max_ttl" {
+  description = "Maximum lease TTL in seconds for keys generated from this role"
+  type        = number
+  default     = null
+}
+
+variable "metadata" {
+  description = "Metadata attached to each generated key"
+  type        = map(string)
+  default     = null
+}

modules/vault_cluster/variables.tf

@@ -289,6 +289,32 @@ variable "kubernetes_secret_backend_role" {
   default = {}
 }
 
+variable "litellm_secret_backend" {
+  description = "Map of LiteLLM secret engines to create (mount + config). The master key is read from KV"
+  type = map(object({
+    plugin                  = optional(string, "vault-plugin-secrets-litellm")
+    description             = optional(string)
+    base_url                = string
+    request_timeout_seconds = optional(number, 30)
+  }))
+  default = {}
+}
+
+variable "litellm_secret_backend_role" {
+  description = "Map of LiteLLM roles to create"
+  type = map(object({
+    name             = string
+    backend          = string
+    models           = optional(list(string))
+    max_budget       = optional(number)
+    key_alias_prefix = optional(string)
+    ttl              = optional(number)
+    max_ttl          = optional(number)
+    metadata         = optional(map(string))
+  }))
+  default = {}
+}
+
 variable "policy_auth_map" {
   description = "Map of auth mounts -> auth roles -> policy names"
   type        = map(map(list(string)))

policies/litellm/admin.yaml

@@ -0,0 +1,26 @@
+# Allow management of the LiteLLM secrets engine (config and roles)
+---
+rules:
+  - path: "litellm/config"
+    capabilities:
+      - create
+      - update
+      - read
+      - delete
+  - path: "litellm/roles/*"
+    capabilities:
+      - create
+      - update
+      - delete
+      - read
+      - list
+  - path: "litellm/roles"
+    capabilities:
+      - read
+      - list
+
+auth:
+  approle:
+    - tf_vault
+  k8s/au/syd1:
+    - woodpecker_terraform_vault