diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d38d4b..646cd65 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,8 +9,8 @@ repos: - repo: https://github.com/gruntwork-io/pre-commit rev: v0.1.30 hooks: - - id: terraform-fmt - - id: terraform-validate + - id: tofu-fmt + - id: tofu-validate - id: tflint - id: terragrunt-hcl-fmt - repo: https://github.com/adrienverge/yamllint.git diff --git a/Makefile b/Makefile index 69ca190..7f5eb89 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ apply: init @terragrunt run --all --parallelism 2 --non-interactive apply format: - @echo "Formatting Terraform files..." - @terraform fmt -recursive . + @echo "Formatting OpenTofu files..." + @tofu fmt -recursive . @echo "Formatting Terragrunt files..." @terragrunt hcl fmt diff --git a/config/config.hcl b/config/config.hcl index 405d97e..e165912 100644 --- a/config/config.hcl +++ b/config/config.hcl @@ -169,7 +169,7 @@ locals { } consul_secret_backend = { for file_path, content in local.all_configs : - trimsuffix(basename(file_path), ".yaml") => content + trimsuffix(replace(file_path, "consul_secret_backend/", ""), ".yaml") => content if startswith(file_path, "consul_secret_backend/") } consul_secret_backend_role = { @@ -186,4 +186,4 @@ locals { if startswith(file_path, "pki_mount_only/") } } -} \ No newline at end of file +} diff --git a/config/consul_secret_backend/consul_root/au/syd1.yaml b/config/consul_secret_backend/consul_root/au/syd1.yaml new file mode 100644 index 0000000..d925a16 --- /dev/null +++ b/config/consul_secret_backend/consul_root/au/syd1.yaml @@ -0,0 +1,7 @@ +description: "consul secret engine for au-syd1 cluster" +default_lease_ttl_seconds: 600 +max_lease_ttl_seconds: 86400 +address: "consul.service.au-syd1.consul" +scheme: https +bootstrap: false +datacenter: au-syd1 diff --git a/config/consul_secret_backend_role/consul_root/au/syd1/terraform-incus.yaml b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-incus.yaml new file mode 100644 index 0000000..608c128 --- /dev/null +++ b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-incus.yaml @@ -0,0 +1,5 @@ +consul_roles: + - terraform-incus +ttl: 300 +max_ttl: 600 +datacenters: [] diff --git a/config/consul_secret_backend_role/consul_root/au/syd1/terraform-k8s.yaml b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-k8s.yaml new file mode 100644 index 0000000..ede5571 --- /dev/null +++ b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-k8s.yaml @@ -0,0 +1,5 @@ +consul_roles: + - terraform-k8s +ttl: 120 +max_ttl: 300 +datacenters: [] diff --git a/config/consul_secret_backend_role/consul_root/au/syd1/terraform-nomad.yaml b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-nomad.yaml new file mode 100644 index 0000000..c13df74 --- /dev/null +++ b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-nomad.yaml @@ -0,0 +1,5 @@ +consul_roles: + - terraform-nomad +ttl: 120 +max_ttl: 300 +datacenters: [] diff --git a/config/consul_secret_backend_role/consul_root/au/syd1/terraform-repoflow.yaml b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-repoflow.yaml new file mode 100644 index 0000000..25189a5 --- /dev/null +++ b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-repoflow.yaml @@ -0,0 +1,5 @@ +consul_roles: + - terraform-repoflow +ttl: 120 +max_ttl: 300 +datacenters: [] diff --git a/config/consul_secret_backend_role/consul_root/au/syd1/terraform-vault.yaml b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-vault.yaml new file mode 100644 index 0000000..dd78b1a --- /dev/null +++ b/config/consul_secret_backend_role/consul_root/au/syd1/terraform-vault.yaml @@ -0,0 +1,5 @@ +consul_roles: + - terraform-vault +ttl: 120 +max_ttl: 300 +datacenters: [] diff --git a/environments/au/syd1/terragrunt.hcl b/environments/au/syd1/terragrunt.hcl index 6549cee..74e7c86 100644 --- a/environments/au/syd1/terragrunt.hcl +++ b/environments/au/syd1/terragrunt.hcl @@ -13,6 +13,11 @@ include "policies" { expose = true } +include "resources" { + path = "${get_repo_root()}/resources/resources.hcl" + expose = true +} + locals { # Extract country and region from path path_parts = split("/", dirname(get_terragrunt_dir())) @@ -24,6 +29,16 @@ locals { # Include policies from policies.hcl policies = include.policies.locals + + # Include resources from resources.hcl + resources = include.resources.locals + + # Create sanitized backend name mapping for Consul providers + # Provider aliases can't contain slashes, so replace them with underscores + consul_backend_aliases = { + for backend_name, _ in local.config.consul_secret_backend : + backend_name => replace(backend_name, "/", "_") + } } terraform { @@ -57,4 +72,7 @@ inputs = { # Pass policy maps to vault_cluster module policy_auth_map = local.policies.policy_auth_map policy_rules_map = local.policies.policy_rules_map + + # Pass sanitized consul backend aliases for provider configuration + consul_backend_aliases = local.consul_backend_aliases } diff --git a/environments/root.hcl b/environments/root.hcl index 43e86bf..5b995f1 100644 --- a/environments/root.hcl +++ b/environments/root.hcl @@ -3,27 +3,14 @@ generate "backend" { path = "backend.tf" if_exists = "overwrite" contents = < -#------------------------------------------------------------------------------ terraform { backend "consul" { address = "https://consul.service.consul" @@ -38,6 +25,10 @@ terraform { source = "hashicorp/vault" version = "5.6.0" } + consul = { + source = "hashicorp/consul" + version = "2.23.0" + } } } EOF diff --git a/modules/vault_cluster/main.tf b/modules/vault_cluster/main.tf index a08f636..92eea0b 100644 --- a/modules/vault_cluster/main.tf +++ b/modules/vault_cluster/main.tf @@ -237,6 +237,29 @@ module "consul_secret_backend" { max_lease_ttl_seconds = each.value.max_lease_ttl_seconds } +# Create data sources for consul backend tokens +data "vault_kv_secret_v2" "consul_backend_configs" { + for_each = { + for k, v in var.consul_secret_backend : k => v + if !v.bootstrap + } + + mount = "kv" + name = "service/vault/${var.country}/${var.region}/secret_backend/${each.key}" +} + +# Create Consul ACL management module +module "consul_acl_management" { + source = "./modules/consul_acl_management" + + country = var.country + region = var.region + consul_backends = var.consul_secret_backend + consul_roles = var.consul_secret_backend_role + consul_backend_aliases = var.consul_backend_aliases +} + +# Create consul secret backend roles (Vault resources only) module "consul_secret_backend_role" { source = "./modules/consul_secret_backend_role" @@ -249,7 +272,7 @@ module "consul_secret_backend_role" { max_ttl = each.value.max_ttl local = each.value.local - depends_on = [module.consul_secret_backend] + depends_on = [module.consul_secret_backend, module.consul_acl_management] } module "kubernetes_secret_backend" { @@ -314,3 +337,4 @@ module "pki_mount_only" { enable_delta = each.value.enable_delta delta_rebuild_interval = each.value.delta_rebuild_interval } + diff --git a/modules/vault_cluster/modules/consul_acl_management/.tflint.hcl b/modules/vault_cluster/modules/consul_acl_management/.tflint.hcl new file mode 100644 index 0000000..8f9177e --- /dev/null +++ b/modules/vault_cluster/modules/consul_acl_management/.tflint.hcl @@ -0,0 +1,7 @@ +rule "terraform_required_providers" { + enabled = false +} + +rule "terraform_required_version" { + enabled = false +} \ No newline at end of file diff --git a/modules/vault_cluster/modules/consul_acl_management/main.tf b/modules/vault_cluster/modules/consul_acl_management/main.tf new file mode 100644 index 0000000..0fa5f00 --- /dev/null +++ b/modules/vault_cluster/modules/consul_acl_management/main.tf @@ -0,0 +1,58 @@ +# Get consul backend tokens from Vault +data "vault_kv_secret_v2" "consul_backend_configs" { + for_each = var.consul_backends + + mount = "kv" + name = "service/vault/${var.country}/${var.region}/secret_backend/${each.key}" +} + +# Create consul provider instances for each consul backend +provider "consul" { + alias = "by_backend" + for_each = var.consul_backend_aliases + + address = var.consul_backends[each.key].address + scheme = var.consul_backends[each.key].scheme + ca_file = "/etc/pki/tls/certs/ca-bundle.crt" + token = data.vault_kv_secret_v2.consul_backend_configs[each.key].data["token"] +} + +# Create Consul ACL policies +resource "consul_acl_policy" "policies" { + for_each = var.consul_roles + + provider = consul.by_backend[each.value.backend] + + name = each.value.name + description = each.value.description != null ? each.value.description : "Auto-generated policy for Vault role ${each.value.name}" + rules = file("${path.module}/../../../../../../../../resources/secret_backend/${each.value.backend}/${each.value.name}.hcl") + datacenters = each.value.datacenters +} + +# Create Consul ACL roles +resource "consul_acl_role" "roles" { + for_each = var.consul_roles + + provider = consul.by_backend[each.value.backend] + + name = each.value.name + description = each.value.description != null ? each.value.description : "Auto-generated role for Vault role ${each.value.name}" + + policies = [consul_acl_policy.policies[each.key].name] + + dynamic "service_identities" { + for_each = each.value.service_identities != null ? each.value.service_identities : [] + content { + service_name = service_identities.value.service_name + datacenters = service_identities.value.datacenters + } + } + + dynamic "node_identities" { + for_each = each.value.node_identities != null ? each.value.node_identities : [] + content { + node_name = node_identities.value.node_name + datacenter = node_identities.value.datacenter + } + } +} diff --git a/modules/vault_cluster/modules/consul_acl_management/outputs.tf b/modules/vault_cluster/modules/consul_acl_management/outputs.tf new file mode 100644 index 0000000..9a73092 --- /dev/null +++ b/modules/vault_cluster/modules/consul_acl_management/outputs.tf @@ -0,0 +1,9 @@ +output "consul_acl_policies" { + description = "Map of created Consul ACL policies" + value = consul_acl_policy.policies +} + +output "consul_acl_roles" { + description = "Map of created Consul ACL roles" + value = consul_acl_role.roles +} \ No newline at end of file diff --git a/modules/vault_cluster/modules/consul_acl_management/variables.tf b/modules/vault_cluster/modules/consul_acl_management/variables.tf new file mode 100644 index 0000000..f5f3410 --- /dev/null +++ b/modules/vault_cluster/modules/consul_acl_management/variables.tf @@ -0,0 +1,46 @@ +variable "consul_backends" { + description = "Map of consul secret backends" + type = map(object({ + address = string + scheme = string + bootstrap = bool + bootstrap_token = optional(string) + ca_cert = optional(string) + client_cert = optional(string) + client_key = optional(string) + })) +} + +variable "consul_roles" { + description = "Map of consul secret backend roles" + type = map(object({ + name = string + backend = string + description = optional(string) + datacenters = optional(list(string)) + service_identities = optional(list(object({ + service_name = string + datacenters = optional(list(string)) + }))) + node_identities = optional(list(object({ + node_name = string + datacenter = string + }))) + })) +} + +variable "consul_backend_aliases" { + description = "Map of consul backend names to sanitized provider aliases" + type = map(string) + default = {} +} + +variable "country" { + description = "Country identifier" + type = string +} + +variable "region" { + description = "Region identifier" + type = string +} \ No newline at end of file diff --git a/modules/vault_cluster/modules/consul_secret_backend/main.tf b/modules/vault_cluster/modules/consul_secret_backend/main.tf index a80253a..2bd0a1f 100644 --- a/modules/vault_cluster/modules/consul_secret_backend/main.tf +++ b/modules/vault_cluster/modules/consul_secret_backend/main.tf @@ -18,4 +18,4 @@ resource "vault_consul_secret_backend" "consul" { client_key = var.client_key default_lease_ttl_seconds = var.default_lease_ttl_seconds max_lease_ttl_seconds = var.max_lease_ttl_seconds -} \ No newline at end of file +} diff --git a/modules/vault_cluster/modules/consul_secret_backend_role/main.tf b/modules/vault_cluster/modules/consul_secret_backend_role/main.tf index f910c1a..2cc374f 100644 --- a/modules/vault_cluster/modules/consul_secret_backend_role/main.tf +++ b/modules/vault_cluster/modules/consul_secret_backend_role/main.tf @@ -1,8 +1,9 @@ +# Create Vault Consul secret backend role resource "vault_consul_secret_backend_role" "role" { backend = var.backend name = var.name - consul_roles = var.consul_roles + consul_roles = [var.name] # Use the role name created by consul_acl_management module ttl = var.ttl max_ttl = var.max_ttl local = var.local -} \ No newline at end of file +} diff --git a/modules/vault_cluster/modules/consul_secret_backend_role/variables.tf b/modules/vault_cluster/modules/consul_secret_backend_role/variables.tf index 503c495..519ab74 100644 --- a/modules/vault_cluster/modules/consul_secret_backend_role/variables.tf +++ b/modules/vault_cluster/modules/consul_secret_backend_role/variables.tf @@ -32,4 +32,5 @@ variable "local" { description = "Whether tokens should be local to the datacenter" type = bool default = false -} \ No newline at end of file +} + diff --git a/modules/vault_cluster/variables.tf b/modules/vault_cluster/variables.tf index 759876e..defe9bc 100644 --- a/modules/vault_cluster/variables.tf +++ b/modules/vault_cluster/variables.tf @@ -226,6 +226,7 @@ variable "consul_secret_backend" { description = optional(string) address = string bootstrap = optional(bool, false) + bootstrap_token = optional(string) scheme = optional(string, "https") ca_cert = optional(string) client_cert = optional(string) @@ -245,10 +246,26 @@ variable "consul_secret_backend_role" { ttl = optional(number) max_ttl = optional(number) local = optional(bool, false) + datacenters = optional(list(string)) + description = optional(string) + service_identities = optional(list(object({ + service_name = string + datacenters = optional(list(string)) + }))) + node_identities = optional(list(object({ + node_name = string + datacenter = string + }))) })) default = {} } +variable "consul_backend_aliases" { + description = "Map of consul backend names to sanitized provider aliases" + type = map(string) + default = {} +} + variable "kubernetes_secret_backend" { description = "Map of Kubernetes secret engines to create" type = map(object({ @@ -287,3 +304,4 @@ variable "policy_rules_map" { }))) default = {} } + diff --git a/policies/consul_root/au/syd1/creds/terraform-incus.yaml b/policies/consul_root/au/syd1/creds/terraform-incus.yaml new file mode 100644 index 0000000..5e77cbb --- /dev/null +++ b/policies/consul_root/au/syd1/creds/terraform-incus.yaml @@ -0,0 +1,10 @@ +# generate credentials for the terraform-incus role in consul +--- +rules: + - path: "consul_root/au/syd1/creds/terraform-incus" + capabilities: + - read + +auth: + approle: + - terraform_incus diff --git a/policies/consul_root/au/syd1/creds/terraform-k8s.yaml b/policies/consul_root/au/syd1/creds/terraform-k8s.yaml new file mode 100644 index 0000000..e43ef2d --- /dev/null +++ b/policies/consul_root/au/syd1/creds/terraform-k8s.yaml @@ -0,0 +1,10 @@ +# generate credentials for the terraform-k8s role in consul +--- +rules: + - path: "consul_root/au/syd1/creds/terraform-k8s" + capabilities: + - read + +auth: + approle: + - terraform_k8s diff --git a/policies/consul_root/au/syd1/creds/terraform-nomad.yaml b/policies/consul_root/au/syd1/creds/terraform-nomad.yaml new file mode 100644 index 0000000..027a310 --- /dev/null +++ b/policies/consul_root/au/syd1/creds/terraform-nomad.yaml @@ -0,0 +1,10 @@ +# generate credentials for the terraform-nomad role in consul +--- +rules: + - path: "consul_root/au/syd1/creds/terraform-nomad" + capabilities: + - read + +auth: + approle: + - terraform_nomad diff --git a/policies/consul_root/au/syd1/creds/terraform-repoflow.yaml b/policies/consul_root/au/syd1/creds/terraform-repoflow.yaml new file mode 100644 index 0000000..5522f65 --- /dev/null +++ b/policies/consul_root/au/syd1/creds/terraform-repoflow.yaml @@ -0,0 +1,10 @@ +# generate credentials for the terraform-repoflow role in consul +--- +rules: + - path: "consul_root/au/syd1/creds/terraform-repoflow" + capabilities: + - read + +auth: + approle: + - terraform_repoflow diff --git a/policies/consul_root/au/syd1/creds/terraform-vault.yaml b/policies/consul_root/au/syd1/creds/terraform-vault.yaml new file mode 100644 index 0000000..0980e77 --- /dev/null +++ b/policies/consul_root/au/syd1/creds/terraform-vault.yaml @@ -0,0 +1,10 @@ +# generate credentials for the terraform-vault role in consul +--- +rules: + - path: "consul_root/au/syd1/creds/terraform-vault" + capabilities: + - read + +auth: + approle: + - tf_vault diff --git a/resources/secret_backend/consul_root/au/syd1/terraform-incus.hcl b/resources/secret_backend/consul_root/au/syd1/terraform-incus.hcl new file mode 100644 index 0000000..4183cd2 --- /dev/null +++ b/resources/secret_backend/consul_root/au/syd1/terraform-incus.hcl @@ -0,0 +1,7 @@ +key_prefix "infra/terraform/incus/" { + policy = "write" +} + +session_prefix "" { + policy = "write" +} diff --git a/resources/secret_backend/consul_root/au/syd1/terraform-k8s.hcl b/resources/secret_backend/consul_root/au/syd1/terraform-k8s.hcl new file mode 100644 index 0000000..f2f1ef6 --- /dev/null +++ b/resources/secret_backend/consul_root/au/syd1/terraform-k8s.hcl @@ -0,0 +1,7 @@ +key_prefix "infra/terraform/k8s/" { + policy = "write" +} + +session_prefix "" { + policy = "write" +} diff --git a/resources/secret_backend/consul_root/au/syd1/terraform-nomad.hcl b/resources/secret_backend/consul_root/au/syd1/terraform-nomad.hcl new file mode 100644 index 0000000..3b1facf --- /dev/null +++ b/resources/secret_backend/consul_root/au/syd1/terraform-nomad.hcl @@ -0,0 +1,7 @@ +key_prefix "infra/terraform/nomad/" { + policy = "write" +} + +session_prefix "" { + policy = "write" +} diff --git a/resources/secret_backend/consul_root/au/syd1/terraform-repoflow.hcl b/resources/secret_backend/consul_root/au/syd1/terraform-repoflow.hcl new file mode 100644 index 0000000..aaab90b --- /dev/null +++ b/resources/secret_backend/consul_root/au/syd1/terraform-repoflow.hcl @@ -0,0 +1,7 @@ +key_prefix "infra/terraform/repoflow/" { + policy = "write" +} + +session_prefix "" { + policy = "write" +} diff --git a/resources/secret_backend/consul_root/au/syd1/terraform-vault.hcl b/resources/secret_backend/consul_root/au/syd1/terraform-vault.hcl new file mode 100644 index 0000000..41e5793 --- /dev/null +++ b/resources/secret_backend/consul_root/au/syd1/terraform-vault.hcl @@ -0,0 +1,11 @@ +key_prefix "infra/terraform/state" { + policy = "write" +} + +key_prefix "infra/terraform/vault/" { + policy = "write" +} + +session_prefix "" { + policy = "write" +} \ No newline at end of file