Merge pull request 'feat: implement consul ACL management with provider aliases' (#48) from benvin/consul_backend into master

Reviewed-on: #48

.pre-commit-config.yaml

@@ -9,8 +9,8 @@ repos:
   - repo: https://github.com/gruntwork-io/pre-commit
     rev: v0.1.30
     hooks:
-      - id: terraform-fmt
+      - id: tofu-fmt
-      - id: terraform-validate
+      - id: tofu-validate
       - id: tflint
       - id: terragrunt-hcl-fmt
   - repo: https://github.com/adrienverge/yamllint.git

Makefile

@@ -23,7 +23,7 @@ apply: init
 	@terragrunt run --all --parallelism 2 --non-interactive apply
 
 format:
-	@echo "Formatting Terraform files..."
+	@echo "Formatting OpenTofu files..."
-	@terraform fmt -recursive .
+	@tofu fmt -recursive .
 	@echo "Formatting Terragrunt files..."
 	@terragrunt hcl fmt

config/config.hcl

@@ -169,7 +169,7 @@ locals {
     }
     consul_secret_backend = {
       for file_path, content in local.all_configs :
-      trimsuffix(basename(file_path), ".yaml") => content
+      trimsuffix(replace(file_path, "consul_secret_backend/", ""), ".yaml") => content
       if startswith(file_path, "consul_secret_backend/")
     }
     consul_secret_backend_role = {
@@ -186,4 +186,4 @@ locals {
       if startswith(file_path, "pki_mount_only/")
     }
   }
-}
+}

config/consul_secret_backend/consul_root/au/syd1.yaml

@@ -0,0 +1,7 @@
+description: "consul secret engine for au-syd1 cluster"
+default_lease_ttl_seconds: 600
+max_lease_ttl_seconds: 86400
+address: "consul.service.au-syd1.consul"
+scheme: https
+bootstrap: false
+datacenter: au-syd1

config/consul_secret_backend_role/consul_root/au/syd1/terraform-incus.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-incus
+ttl: 300
+max_ttl: 600
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-k8s.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-k8s
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-nomad.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-nomad
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-repoflow.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-repoflow
+ttl: 120
+max_ttl: 300
+datacenters: []

config/consul_secret_backend_role/consul_root/au/syd1/terraform-vault.yaml

@@ -0,0 +1,5 @@
+consul_roles:
+  - terraform-vault
+ttl: 120
+max_ttl: 300
+datacenters: []

environments/au/syd1/terragrunt.hcl

@@ -13,6 +13,11 @@ include "policies" {
   expose = true
 }
 
+include "resources" {
+  path   = "${get_repo_root()}/resources/resources.hcl"
+  expose = true
+}
+
 locals {
   # Extract country and region from path
   path_parts = split("/", dirname(get_terragrunt_dir()))
@@ -24,6 +29,16 @@ locals {
 
   # Include policies from policies.hcl
   policies = include.policies.locals
+
+  # Include resources from resources.hcl
+  resources = include.resources.locals
+
+  # Create sanitized backend name mapping for Consul providers
+  # Provider aliases can't contain slashes, so replace them with underscores
+  consul_backend_aliases = {
+    for backend_name, _ in local.config.consul_secret_backend :
+    backend_name => replace(backend_name, "/", "_")
+  }
 }
 
 terraform {
@@ -57,4 +72,7 @@ inputs = {
   # Pass policy maps to vault_cluster module
   policy_auth_map  = local.policies.policy_auth_map
   policy_rules_map = local.policies.policy_rules_map
+
+  # Pass sanitized consul backend aliases for provider configuration
+  consul_backend_aliases = local.consul_backend_aliases
 }

environments/root.hcl

@@ -3,27 +3,14 @@ generate "backend" {
   path      = "backend.tf"
   if_exists = "overwrite"
   contents  = <<EOF
-#-------------------------------------------
-# locals
-#-------------------------------------------
 locals {
   vault_addr = "https://vault.service.consul:8200"
 }
 
-#-----------------------------------------------------------------------------
-# Configure this provider through the environment variables:
-#    - VAULT_ADDR
-#    - VAULT_TOKEN
-#-----------------------------------------------------------------------------
 provider "vault" {
   address = local.vault_addr
 }
 
-#------------------------------------------------------------------------------
-# Use remote state file and encrypt it since your state files may contains
-# sensitive data.
-# export CONSUL_HTTP_TOKEN=<your-token>
-#------------------------------------------------------------------------------
 terraform {
   backend "consul" {
     address = "https://consul.service.consul"
@@ -38,6 +25,10 @@ terraform {
       source  = "hashicorp/vault"
       version = "5.6.0"
     }
+    consul = {
+      source  = "hashicorp/consul"
+      version = "2.23.0"
+    }
   }
 }
 EOF

modules/vault_cluster/main.tf

@@ -237,6 +237,29 @@ module "consul_secret_backend" {
   max_lease_ttl_seconds     = each.value.max_lease_ttl_seconds
 }
 
+# Create data sources for consul backend tokens
+data "vault_kv_secret_v2" "consul_backend_configs" {
+  for_each = {
+    for k, v in var.consul_secret_backend : k => v
+    if !v.bootstrap
+  }
+
+  mount = "kv"
+  name  = "service/vault/${var.country}/${var.region}/secret_backend/${each.key}"
+}
+
+# Create Consul ACL management module
+module "consul_acl_management" {
+  source = "./modules/consul_acl_management"
+
+  country                = var.country
+  region                 = var.region
+  consul_backends        = var.consul_secret_backend
+  consul_roles           = var.consul_secret_backend_role
+  consul_backend_aliases = var.consul_backend_aliases
+}
+
+# Create consul secret backend roles (Vault resources only)
 module "consul_secret_backend_role" {
   source = "./modules/consul_secret_backend_role"
 
@@ -249,7 +272,7 @@ module "consul_secret_backend_role" {
   max_ttl      = each.value.max_ttl
   local        = each.value.local
 
-  depends_on = [module.consul_secret_backend]
+  depends_on = [module.consul_secret_backend, module.consul_acl_management]
 }
 
 module "kubernetes_secret_backend" {
@@ -314,3 +337,4 @@ module "pki_mount_only" {
   enable_delta                  = each.value.enable_delta
   delta_rebuild_interval        = each.value.delta_rebuild_interval
 }
+

modules/vault_cluster/modules/consul_acl_management/.tflint.hcl

@@ -0,0 +1,7 @@
+rule "terraform_required_providers" {
+  enabled = false
+}
+
+rule "terraform_required_version" {
+  enabled = false
+}

modules/vault_cluster/modules/consul_acl_management/main.tf

@@ -0,0 +1,58 @@
+# Get consul backend tokens from Vault
+data "vault_kv_secret_v2" "consul_backend_configs" {
+  for_each = var.consul_backends
+
+  mount = "kv"
+  name  = "service/vault/${var.country}/${var.region}/secret_backend/${each.key}"
+}
+
+# Create consul provider instances for each consul backend
+provider "consul" {
+  alias    = "by_backend"
+  for_each = var.consul_backend_aliases
+
+  address = var.consul_backends[each.key].address
+  scheme  = var.consul_backends[each.key].scheme
+  ca_file = "/etc/pki/tls/certs/ca-bundle.crt"
+  token   = data.vault_kv_secret_v2.consul_backend_configs[each.key].data["token"]
+}
+
+# Create Consul ACL policies
+resource "consul_acl_policy" "policies" {
+  for_each = var.consul_roles
+
+  provider = consul.by_backend[each.value.backend]
+
+  name        = each.value.name
+  description = each.value.description != null ? each.value.description : "Auto-generated policy for Vault role ${each.value.name}"
+  rules       = file("${path.module}/../../../../../../../../resources/secret_backend/${each.value.backend}/${each.value.name}.hcl")
+  datacenters = each.value.datacenters
+}
+
+# Create Consul ACL roles
+resource "consul_acl_role" "roles" {
+  for_each = var.consul_roles
+
+  provider = consul.by_backend[each.value.backend]
+
+  name        = each.value.name
+  description = each.value.description != null ? each.value.description : "Auto-generated role for Vault role ${each.value.name}"
+
+  policies = [consul_acl_policy.policies[each.key].name]
+
+  dynamic "service_identities" {
+    for_each = each.value.service_identities != null ? each.value.service_identities : []
+    content {
+      service_name = service_identities.value.service_name
+      datacenters  = service_identities.value.datacenters
+    }
+  }
+
+  dynamic "node_identities" {
+    for_each = each.value.node_identities != null ? each.value.node_identities : []
+    content {
+      node_name  = node_identities.value.node_name
+      datacenter = node_identities.value.datacenter
+    }
+  }
+}

modules/vault_cluster/modules/consul_acl_management/outputs.tf

@@ -0,0 +1,9 @@
+output "consul_acl_policies" {
+  description = "Map of created Consul ACL policies"
+  value       = consul_acl_policy.policies
+}
+
+output "consul_acl_roles" {
+  description = "Map of created Consul ACL roles"
+  value       = consul_acl_role.roles
+}

modules/vault_cluster/modules/consul_acl_management/variables.tf

@@ -0,0 +1,46 @@
+variable "consul_backends" {
+  description = "Map of consul secret backends"
+  type = map(object({
+    address         = string
+    scheme          = string
+    bootstrap       = bool
+    bootstrap_token = optional(string)
+    ca_cert         = optional(string)
+    client_cert     = optional(string)
+    client_key      = optional(string)
+  }))
+}
+
+variable "consul_roles" {
+  description = "Map of consul secret backend roles"
+  type = map(object({
+    name        = string
+    backend     = string
+    description = optional(string)
+    datacenters = optional(list(string))
+    service_identities = optional(list(object({
+      service_name = string
+      datacenters  = optional(list(string))
+    })))
+    node_identities = optional(list(object({
+      node_name  = string
+      datacenter = string
+    })))
+  }))
+}
+
+variable "consul_backend_aliases" {
+  description = "Map of consul backend names to sanitized provider aliases"
+  type        = map(string)
+  default     = {}
+}
+
+variable "country" {
+  description = "Country identifier"
+  type        = string
+}
+
+variable "region" {
+  description = "Region identifier"
+  type        = string
+}

modules/vault_cluster/modules/consul_secret_backend/main.tf

@@ -18,4 +18,4 @@ resource "vault_consul_secret_backend" "consul" {
   client_key                = var.client_key
   default_lease_ttl_seconds = var.default_lease_ttl_seconds
   max_lease_ttl_seconds     = var.max_lease_ttl_seconds
-}
+}

modules/vault_cluster/modules/consul_secret_backend_role/main.tf

@@ -1,8 +1,9 @@
+# Create Vault Consul secret backend role
 resource "vault_consul_secret_backend_role" "role" {
   backend      = var.backend
   name         = var.name
-  consul_roles = var.consul_roles
+  consul_roles = [var.name] # Use the role name created by consul_acl_management module
   ttl          = var.ttl
   max_ttl      = var.max_ttl
   local        = var.local
-}
+}

modules/vault_cluster/modules/consul_secret_backend_role/variables.tf

@@ -32,4 +32,5 @@ variable "local" {
   description = "Whether tokens should be local to the datacenter"
   type        = bool
   default     = false
-}
+}
+

modules/vault_cluster/variables.tf

@@ -226,6 +226,7 @@ variable "consul_secret_backend" {
     description               = optional(string)
     address                   = string
     bootstrap                 = optional(bool, false)
+    bootstrap_token           = optional(string)
     scheme                    = optional(string, "https")
     ca_cert                   = optional(string)
     client_cert               = optional(string)
@@ -245,10 +246,26 @@ variable "consul_secret_backend_role" {
     ttl          = optional(number)
     max_ttl      = optional(number)
     local        = optional(bool, false)
+    datacenters  = optional(list(string))
+    description  = optional(string)
+    service_identities = optional(list(object({
+      service_name = string
+      datacenters  = optional(list(string))
+    })))
+    node_identities = optional(list(object({
+      node_name  = string
+      datacenter = string
+    })))
   }))
   default = {}
 }
 
+variable "consul_backend_aliases" {
+  description = "Map of consul backend names to sanitized provider aliases"
+  type        = map(string)
+  default     = {}
+}
+
 variable "kubernetes_secret_backend" {
   description = "Map of Kubernetes secret engines to create"
   type = map(object({
@@ -287,3 +304,4 @@ variable "policy_rules_map" {
   })))
   default = {}
 }
+

policies/consul_root/au/syd1/creds/terraform-incus.yaml

@@ -0,0 +1,10 @@
+# generate credentials for the terraform-incus role in consul
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-incus"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_incus

policies/consul_root/au/syd1/creds/terraform-k8s.yaml

@@ -0,0 +1,10 @@
+# generate credentials for the terraform-k8s role in consul
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-k8s"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_k8s

policies/consul_root/au/syd1/creds/terraform-nomad.yaml

@@ -0,0 +1,10 @@
+# generate credentials for the terraform-nomad role in consul
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-nomad"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_nomad

policies/consul_root/au/syd1/creds/terraform-repoflow.yaml

@@ -0,0 +1,10 @@
+# generate credentials for the terraform-repoflow role in consul
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-repoflow"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - terraform_repoflow

policies/consul_root/au/syd1/creds/terraform-vault.yaml

@@ -0,0 +1,10 @@
+# generate credentials for the terraform-vault role in consul
+---
+rules:
+  - path: "consul_root/au/syd1/creds/terraform-vault"
+    capabilities:
+      - read
+
+auth:
+  approle:
+    - tf_vault

resources/secret_backend/consul_root/au/syd1/terraform-incus.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/incus/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-k8s.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/k8s/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-nomad.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/nomad/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-repoflow.hcl

@@ -0,0 +1,7 @@
+key_prefix "infra/terraform/repoflow/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}

resources/secret_backend/consul_root/au/syd1/terraform-vault.hcl

@@ -0,0 +1,11 @@
+key_prefix "infra/terraform/state" {
+  policy = "write"
+}
+
+key_prefix "infra/terraform/vault/" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "write"
+}