feat: major restructuring in migration to terragrunt

- migrate from individual terraform files to config-driven terragrunt module structure - add vault_cluster module with config discovery system - replace individual .tf files with centralized config.hcl - restructure auth and secret backends as configurable modules - move auth roles and secret backends to yaml-based configuration - convert policies from .hcl to .yaml format, add rules/auth definition - add pre-commit hooks for yaml formatting and file cleanup - add terragrunt cache to gitignore - update makefile with terragrunt commands and format target
2026-01-04 23:31:42 +11:00
parent bd112181f5
commit 8070b6f66b
245 changed files with 3943 additions and 985 deletions
@@ -0,0 +1,2 @@
+default_lease_ttl: 60s
+max_lease_ttl: 24h
@@ -0,0 +1,13 @@
+token_policies:
+  - "pki_int/certmanager"
+token_ttl: 30
+token_max_ttl: 30
+bind_secret_id: false
+token_bound_cidrs:
+  - "198.18.25.5/32"
+  - "198.18.26.3/32"
+  - "198.18.27.89/32"
+  - "198.18.28.8/32"
+  - "198.18.29.33/32"
+  - "198.18.29.239/32"
+use_deterministic_role_id: false
@@ -0,0 +1,12 @@
+token_policies:
+  - "default_access"
+  - "kv/service/incus/incus-cluster-join-tokens"
+token_ttl: 60
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.13.77/32"
+  - "198.18.13.78/32"
+  - "198.18.13.79/32"
+use_deterministic_role_id: false
@@ -0,0 +1,12 @@
+token_policies:
+  - "default_access"
+  - "kv/service/packer/packer_builder"
+token_ttl: 300
+token_max_ttl: 600
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: false
@@ -0,0 +1,13 @@
+token_policies:
+  - "kv/service/puppetapi/puppetapi_read_tokens"
+token_ttl: 30
+token_max_ttl: 30
+bind_secret_id: false
+token_bound_cidrs:
+  - "198.18.25.5/32"
+  - "198.18.26.3/32"
+  - "198.18.27.89/32"
+  - "198.18.28.8/32"
+  - "198.18.29.33/32"
+  - "198.18.29.239/32"
+use_deterministic_role_id: false
@@ -0,0 +1,12 @@
+token_policies:
+  - "kv/service/github/neoloc/tokens/read-only-token/read"
+  - "kv/service/gitea/unkinben/tokens/read-only-packages/read"
+token_ttl: 30
+token_max_ttl: 30
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: false
@@ -0,0 +1,8 @@
+token_policies:
+  - "rundeck/rundeck"
+token_ttl: 3600
+token_max_ttl: 14400
+bind_secret_id: true
+token_bound_cidrs:
+  - "198.18.13.59/32"
+use_deterministic_role_id: false
@@ -0,0 +1,14 @@
+token_policies:
+  - "ssh-host-signer/sshsigner"
+  - "sshca_signhost"
+token_ttl: 30
+token_max_ttl: 30
+bind_secret_id: false
+token_bound_cidrs:
+  - "198.18.25.5/32"
+  - "198.18.26.3/32"
+  - "198.18.27.89/32"
+  - "198.18.28.8/32"
+  - "198.18.29.33/32"
+  - "198.18.29.239/32"
+use_deterministic_role_id: false
@@ -0,0 +1,13 @@
+token_policies:
+  - "default_access"
+  - "kv/service/terraform/incus"
+  - "kv/service/puppet/certificates/terraform_puppet_cert"
+token_ttl: 60
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: false
@@ -0,0 +1,12 @@
+token_policies:
+  - "default_access"
+  - "kv/service/terraform/nomad"
+token_ttl: 60
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: false
@@ -0,0 +1,13 @@
+token_policies:
+  - "default_access"
+  - "kv/service/repoflow/unkinadmin/tokens/terraform/read"
+  - "kv/service/terraform/repoflow"
+token_ttl: 60
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+  - "198.18.25.102/32"
+  - "198.18.26.91/32"
+  - "198.18.27.40/32"
+use_deterministic_role_id: false
@@ -0,0 +1,30 @@
+token_policies:
+  - "default_access"
+  - "approle_token_create"
+  - "auth/approle/approle_role_admin"
+  - "auth/approle/approle_role_login"
+  - "auth/kubernetes/k8s_auth_admin"
+  - "auth/ldap/ldap_admin"
+  - "auth/token/auth_token_create"
+  - "auth/token/auth_token_self"
+  - "auth/token/auth_token_roles_admin"
+  - "kubernetes/au/config_admin"
+  - "kubernetes/au/roles_admin"
+  - "kv/service/glauth/services/svc_vault_read"
+  - "kv/service/kubernetes/au/syd1/token_reviewer_jwt/read"
+  - "kv/service/kubernetes/au/syd1/service_account_jwt/read"
+  - "kv/service/vault/auth_backends_read"
+  - "pki_int/pki_int_roles_admin"
+  - "pki_root/pki_root_roles_admin"
+  - "ssh-host-signer/ssh-host-signer_roles_admin"
+  - "sshca/sshca_roles_admin"
+  - "sys/sys_auth_admin"
+  - "sys/sys_mounts_admin"
+  - "sys/sys_policy_admin"
+  - "transit/keys/admin"
+token_ttl: 60
+token_max_ttl: 120
+bind_secret_id: false
+token_bound_cidrs:
+  - "10.10.12.200/32"
+use_deterministic_role_id: false
@@ -0,0 +1,5 @@
+kubernetes_host: https://api-k8s.service.consul:6443
+disable_iss_validation: true
+use_annotations_as_alias_metadata: true
+default_lease_ttl: 1h
+max_lease_ttl: 24h
@@ -0,0 +1,11 @@
+bound_service_account_names:
+  - ceph-csi-rbd-csi-rbd-provisioner
+  - ceph-csi-cephfs-csi-cephfs-provisioner
+bound_service_account_namespaces:
+  - csi-cephrbd
+  - csi-cephfs
+token_ttl: 60
+token_policies:
+  - kv/service/kubernetes/au/syd1/csi/ceph-rbd-secret/read
+  - kv/service/kubernetes/au/syd1/csi/ceph-cephfs-secret/read
+audience: vault
@@ -0,0 +1,9 @@
+bound_service_account_names:
+  - cert-manager-vault-issuer
+bound_service_account_namespaces:
+  - cert-manager
+token_ttl: 60
+token_policies:
+  - pki_int/sign/servers_default
+  - pki_int/issue/servers_default
+audience: vault
@@ -0,0 +1,8 @@
+bound_service_account_names:
+  - externaldns
+bound_service_account_namespaces:
+  - externaldns
+token_ttl: 60
+token_policies:
+  - kv/service/kubernetes/au/syd1/externaldns/tsig/read
+audience: vault
@@ -0,0 +1,9 @@
+bound_service_account_names:
+  - default
+bound_service_account_namespaces:
+  - huntarr
+token_ttl: 60
+token_policies:
+  - pki_int/sign/servers_default
+  - pki_int/issue/servers_default
+audience: vault
@@ -0,0 +1,9 @@
+bound_service_account_names:
+  - media-apps-vault-reader
+bound_service_account_namespaces:
+  - media-apps
+token_ttl: 60
+token_policies:
+  - kv/service/media-apps/radarr/read
+  - kv/service/media-apps/sonarr/read
+audience: vault
@@ -0,0 +1,12 @@
+bound_service_account_names:
+  - default
+bound_service_account_namespaces:
+  - repoflow
+token_ttl: 60
+token_policies:
+  - kv/service/repoflow/au/syd1/ceph-s3/read
+  - kv/service/repoflow/au/syd1/elasticsearch/read
+  - kv/service/repoflow/au/syd1/hasura/read
+  - kv/service/repoflow/au/syd1/postgres/read
+  - kv/service/repoflow/au/syd1/repoflow-server/read
+audience: vault
@@ -0,0 +1,10 @@
+userdn: "ou=people,ou=users,dc=main,dc=unkin,dc=net"
+userattr: "uid"
+upndomain: "users.main.unkin.net"
+discoverdn: false
+groupdn: "ou=users,dc=main,dc=unkin,dc=net"
+groupfilter: "(&(objectClass=posixGroup)(memberUid={{.Username}}))"
+groupattr: "uid"
+username_as_alias: true
+default_lease_ttl: 24h
+max_lease_ttl: 168h
@@ -0,0 +1,2 @@
+policies:
+  - "default_access"
@@ -0,0 +1,3 @@
+policies:
+  - "default_access"
+  - "global-admin"
@@ -0,0 +1,189 @@
+# =============================================================================
+# VAULT MODULE CONFIGURATION SYSTEM
+# =============================================================================
+#
+# This file automatically discovers and organizes YAML configuration files
+# for Vault modules, creating structured configuration maps for Terraform.
+#
+# HOW IT WORKS:
+# 1. Scans all subdirectories for *.yaml files
+# 2. Groups files by module type based on directory structure
+# 3. Creates unique resource keys to prevent naming conflicts
+# 4. Adds computed fields like name, backend, etc. from file paths
+#
+# DIRECTORY STRUCTURE:
+# config/
+# ├── auth_approle_role/
+# │   └── approle/
+# │       ├── certmanager.yaml    # Creates key: "approle/certmanager"
+# │       └── myapp.yaml          # Creates key: "approle/myapp"
+# ├── auth_kubernetes_role/
+# │   └── k8s/au/syd1/
+# │       ├── default.yaml        # Creates key: "k8s/au/syd1/default"
+# │       └── myapp.yaml          # Creates key: "k8s/au/syd1/myapp"
+# └── kv_secret_backend/
+#     ├── kv.yaml                 # Creates key: "kv"
+#     └── secrets.yaml            # Creates key: "secrets"
+#
+# EXAMPLE YAML FILE (config/auth_approle_role/approle/myapp.yaml):
+# ```yaml
+# token_ttl: 3600
+# token_max_ttl: 7200
+# bind_secret_id: true
+# token_bound_cidrs:
+#   - "10.0.0.0/8"
+# ```
+#
+# This becomes:
+# ```hcl
+# auth_approle_role = {
+#   "approle/myapp" = {
+#     approle_name = "myapp"        # Auto-computed from filename
+#     mount_path = "approle"        # Auto-computed from directory
+#     token_ttl = 3600              # From YAML content
+#     token_max_ttl = 7200          # From YAML content
+#     bind_secret_id = true         # From YAML content
+#     token_bound_cidrs = ["10.0.0.0/8"]
+#   }
+# }
+# ```
+#
+# KEY NAMING PATTERNS:
+# - Simple backends: filename only (e.g., "kv", "transit")
+# - Role-based resources: full path without extension (e.g., "approle/myapp")
+# - This ensures uniqueness when multiple backends have similar role names
+#
+# GENERATED OUTPUTS:
+# - config.auth_approle_backend, config.auth_approle_role, etc.
+# - Each module gets its own map with properly structured configuration
+#
+# =============================================================================
+
+locals {
+  # Find all YAML files in subdirectories
+  config_files = fileset(".", "**/*.yaml")
+
+  # Create a flat map of all files with their content
+  all_configs = {
+    for file_path in local.config_files :
+    file_path => yamldecode(file(file_path))
+  }
+
+  # Group by module directory (first part of path)
+  config = {
+    auth_approle_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "auth_approle_backend/")
+    }
+    auth_approle_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "auth_approle_role/", ""), ".yaml") => merge(content, {
+        approle_name = trimsuffix(basename(file_path), ".yaml")
+        mount_path   = split("/", replace(file_path, "auth_approle_role/", ""))[0]
+      })
+      if startswith(file_path, "auth_approle_role/")
+    }
+    auth_ldap_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "auth_ldap_backend/")
+    }
+    auth_ldap_group = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "auth_ldap_group/", ""), ".yaml") => merge(content, {
+        groupname = trimsuffix(basename(file_path), ".yaml")
+        backend   = split("/", replace(file_path, "auth_ldap_group/", ""))[0]
+      })
+      if startswith(file_path, "auth_ldap_group/")
+    }
+    auth_kubernetes_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "auth_kubernetes_backend/", ""), ".yaml") => content
+      if startswith(file_path, "auth_kubernetes_backend/")
+    }
+    auth_kubernetes_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "auth_kubernetes_role/", ""), ".yaml") => merge(content, {
+        role_name = trimsuffix(basename(file_path), ".yaml")
+        backend   = dirname(replace(file_path, "auth_kubernetes_role/", ""))
+      })
+      if startswith(file_path, "auth_kubernetes_role/")
+    }
+    kv_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "kv_secret_backend/")
+    }
+    transit_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "transit_secret_backend/")
+    }
+    transit_secret_backend_key = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "transit_secret_backend_key/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "transit_secret_backend_key/", ""))
+      })
+      if startswith(file_path, "transit_secret_backend_key/")
+    }
+    ssh_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "ssh_secret_backend/")
+    }
+    ssh_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "ssh_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "ssh_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "ssh_secret_backend_role/")
+    }
+    pki_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "pki_secret_backend/", ""), ".yaml") => content
+      if startswith(file_path, "pki_secret_backend/")
+    }
+    pki_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "pki_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "pki_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "pki_secret_backend_role/")
+    }
+    kubernetes_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "kubernetes_secret_backend/", ""), ".yaml") => content
+      if startswith(file_path, "kubernetes_secret_backend/")
+    }
+    kubernetes_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "kubernetes_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "kubernetes_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "kubernetes_secret_backend_role/")
+    }
+    consul_secret_backend = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "consul_secret_backend/")
+    }
+    consul_secret_backend_role = {
+      for file_path, content in local.all_configs :
+      trimsuffix(replace(file_path, "consul_secret_backend_role/", ""), ".yaml") => merge(content, {
+        name    = trimsuffix(basename(file_path), ".yaml")
+        backend = dirname(replace(file_path, "consul_secret_backend_role/", ""))
+      })
+      if startswith(file_path, "consul_secret_backend_role/")
+    }
+    pki_mount_only = {
+      for file_path, content in local.all_configs :
+      trimsuffix(basename(file_path), ".yaml") => content
+      if startswith(file_path, "pki_mount_only/")
+    }
+  }
+}
@@ -0,0 +1,5 @@
+description: "kubernetes secret engine for au-syd1 cluster"
+default_lease_ttl_seconds: 600
+max_lease_ttl_seconds: 86400
+kubernetes_host: "https://api-k8s.service.consul:6443"
+disable_local_ca_jwt: false
@@ -0,0 +1,5 @@
+backend: "kubernetes/au/syd1"
+allowed_kubernetes_namespaces:
+  - "*"
+kubernetes_role_type: "ClusterRole"
+extra_labels: {}
@@ -0,0 +1,5 @@
+backend: "kubernetes/au/syd1"
+allowed_kubernetes_namespaces:
+  - "*"
+kubernetes_role_type: "ClusterRole"
+extra_labels: {}
@@ -0,0 +1,5 @@
+backend: "kubernetes/au/syd1"
+allowed_kubernetes_namespaces:
+  - "*"
+kubernetes_role_type: "ClusterRole"
+extra_labels: {}
@@ -0,0 +1,5 @@
+backend: "kubernetes/au/syd1"
+allowed_kubernetes_namespaces:
+  - "media-apps"
+kubernetes_role_type: "Role"
+extra_labels: {}
@@ -0,0 +1,4 @@
+type: kv-v2
+description: "Key-Value secrets engine"
+version: "2"
+max_versions: 10
@@ -0,0 +1,4 @@
+type: kv-v2
+description: "Rundeck secrets engine"
+version: "2"
+max_versions: 5
@@ -0,0 +1,17 @@
+description: "PKI Intermediate CA"
+max_lease_ttl_seconds: 157680000  # 43800 hours * 3600
+issuer_ref: "default"
+issuing_certificates:
+  - "https://vault.service.consul:8200/v1/pki_int/ca"
+crl_distribution_points:
+  - "https://vault.service.consul:8200/v1/pki_int/crl"
+ocsp_servers: []
+enable_templating: false
+default_issuer_ref: null
+default_follows_latest_issuer: false
+crl_expiry: "72h"
+crl_disable: false
+ocsp_disable: false
+auto_rebuild: false
+enable_delta: false
+delta_rebuild_interval: null
@@ -0,0 +1,17 @@
+description: "PKI Root CA"
+max_lease_ttl_seconds: 315360000  # 10 years
+issuer_ref: "default"
+issuing_certificates:
+  - "https://vault.service.consul:8200/v1/pki_root/ca"
+crl_distribution_points:
+  - "https://vault.service.consul:8200/v1/pki_root/crl"
+ocsp_servers: []
+enable_templating: false
+default_issuer_ref: null
+default_follows_latest_issuer: false
+crl_expiry: "72h"
+crl_disable: false
+ocsp_disable: false
+auto_rebuild: false
+enable_delta: false
+delta_rebuild_interval: null
@@ -0,0 +1,18 @@
+description: "PKI Root CA AU SYD1"
+max_lease_ttl_seconds: 315360000  # 87600 * 3600
+common_name: "unkin.net AU SYD1 Root CA"
+issuer_name: "UNKIN_AU_SYD1_ROOTCA_2024"
+ttl: 315360000  # 87600 * 3600
+format: "pem"
+issuing_certificates:
+  - "https://vault.service.consul:8200/v1/pki/au/syd1/ca"
+crl_distribution_points:
+  - "https://vault.service.consul:8200/v1/pki/au/syd1/crl"
+ocsp_servers: []
+enable_templating: false
+default_follows_latest_issuer: false
+crl_expiry: "72h"
+crl_disable: false
+ocsp_disable: false
+auto_rebuild: false
+enable_delta: false
@@ -0,0 +1,17 @@
+backend: "pki/au/syd1"
+allow_ip_sans: true
+allowed_domains:
+  - "unkin.net"
+  - "*.unkin.net"
+  - "localhost"
+allow_subdomains: true
+allow_glob_domains: true
+allow_bare_domains: true
+enforce_hostnames: true
+allow_any_name: true
+max_ttl: 7776000  # 2160 * 3600
+key_bits: 4096
+country:
+  - "Australia"
+use_csr_common_name: true
+use_csr_sans: true
@@ -0,0 +1,17 @@
+backend: "pki_int"
+allow_ip_sans: true
+allowed_domains:
+  - "unkin.net"
+  - "*.unkin.net"
+  - "localhost"
+allow_subdomains: true
+allow_glob_domains: true
+allow_bare_domains: true
+enforce_hostnames: true
+allow_any_name: true
+max_ttl: 7776000  # 2160 * 3600
+key_bits: 4096
+country:
+  - "Australia"
+use_csr_common_name: true
+use_csr_sans: true
@@ -0,0 +1,15 @@
+backend: "pki_root"
+allow_ip_sans: true
+allowed_domains:
+  - "unkin.net"
+  - "unkin.local"
+allow_subdomains: true
+allow_glob_domains: false
+allow_bare_domains: true
+enforce_hostnames: false
+allow_any_name: false
+max_ttl: 31536000  # 8760h in seconds
+key_bits: 2048
+country: []
+use_csr_common_name: true
+use_csr_sans: true
@@ -0,0 +1,4 @@
+description: "SSH CA Engine"
+max_lease_ttl_seconds: 315360000  # 87600 * 3600
+generate_signing_key: true
+key_type: ssh-rsa
@@ -0,0 +1,8 @@
+key_type: ca
+algorithm_signer: rsa-sha2-256
+ttl: 315360000  # 87600 * 3600
+allow_host_certificates: true
+allow_user_certificates: false
+allowed_domains: "main.unkin.net,consul"
+allow_subdomains: true
+allow_bare_domains: false
@@ -0,0 +1,3 @@
+description: "Transit Engine"
+default_lease_ttl_seconds: 3600
+max_lease_ttl_seconds: 86400
@@ -0,0 +1,5 @@
+type: aes256-gcm96
+deletion_allowed: false
+derived: false
+exportable: false
+allow_plaintext_backup: false