Dual-write ENC data into encapi alongside Cobbler

Seed encapi (statuses, roles, prodnxsr node assignments) and make the
instance module also create an encapi_node for every VM, so the k8s
puppetserver ENC can cut over from Cobbler. Cobbler/puppetca/puppetdb
resources are left untouched.

- Add config/encapi leaf + modules/encapi driving statuses/roles/nodes
  from YAML (3 statuses, 51 roles, 13 prodnxsr physical nodes).
- Add encapi_node to modules/instance (certname, role, environment).
- Wire encapi provider into root.hcl required_providers and the module
  providers; plumb ENCAPI_WRITE_TOKEN via Makefile vault_env.

Makefile

@@ -10,6 +10,7 @@ define vault_env
 	export PUPPET_CERT_CA=$$(vault kv get -field=public_key kv/service/puppet/certificates/ca) && \
 	export PUPPET_CERT_PUB=$$(vault kv get -field=public_key kv/service/puppet/certificates/terraform) && \
 	export PUPPET_CERT_PRIV=$$(vault kv get -field=private_key kv/service/puppet/certificates/terraform) && \
+	export ENCAPI_WRITE_TOKEN=$$(vault kv get -field=ENCAPI_WRITE_TOKEN kv/kubernetes/namespace/encapi/default/environment) && \
 	export TG_QUEUE_EXCLUDE_DIR="templates/base" && \
 	export TG_PROVIDER_CACHE=1 && \
 	export TG_TF_PATH=terraform && \

config/encapi/nodes.yaml

@@ -0,0 +1,44 @@
+# encapi node assignments for the 13 prodnxsr* physical nodes.
+# These hosts are NOT managed by the incus instance module (they are the
+# bare-metal hypervisors / k8s nodes), so they are seeded here explicitly.
+# certname -> role from the PuppetDB enc_role fact; environment=develop to
+# match their current catalog_environment.
+prodnxsr0001.main.unkin.net:
+  role: roles::infra::k8s::control
+  environment: develop
+prodnxsr0002.main.unkin.net:
+  role: roles::infra::k8s::control
+  environment: develop
+prodnxsr0003.main.unkin.net:
+  role: roles::infra::k8s::control
+  environment: develop
+prodnxsr0004.main.unkin.net:
+  role: roles::infra::k8s::compute
+  environment: develop
+prodnxsr0005.main.unkin.net:
+  role: roles::infra::k8s::compute
+  environment: develop
+prodnxsr0006.main.unkin.net:
+  role: roles::infra::k8s::compute
+  environment: develop
+prodnxsr0007.main.unkin.net:
+  role: roles::infra::k8s::compute
+  environment: develop
+prodnxsr0008.main.unkin.net:
+  role: roles::infra::k8s::compute
+  environment: develop
+prodnxsr0009.main.unkin.net:
+  role: roles::infra::incus::node
+  environment: develop
+prodnxsr0010.main.unkin.net:
+  role: roles::infra::incus::node
+  environment: develop
+prodnxsr0011.main.unkin.net:
+  role: roles::infra::incus::node
+  environment: develop
+prodnxsr0012.main.unkin.net:
+  role: roles::infra::incus::node
+  environment: develop
+prodnxsr0013.main.unkin.net:
+  role: roles::infra::incus::node
+  environment: develop

config/encapi/roles.yaml

@@ -0,0 +1,55 @@
+# encapi roles: every distinct Puppet role class used across
+# config/instances/*/config.yaml (cobbler_mgmt_classes[0]) PLUS the roles
+# run by the 13 prodnxsr physical nodes (from PuppetDB enc_role fact).
+# Values are empty maps; add description/default_params here when needed.
+roles::apps::jupyter::hub: {}
+roles::apps::media::jellyfin: {}
+roles::apps::media::lidarr: {}
+roles::apps::media::nzbget: {}
+roles::apps::media::prowlarr: {}
+roles::apps::media::radarr: {}
+roles::apps::media::readarr: {}
+roles::apps::media::sonarr: {}
+roles::apps::music::gonic: {}
+roles::base: {}
+roles::infra::auth::glauth: {}
+roles::infra::ceph::rgw: {}
+roles::infra::cobbler::server: {}
+roles::infra::dhcp::server: {}
+roles::infra::dns::externaldns: {}
+roles::infra::dns::master: {}
+roles::infra::dns::resolver: {}
+roles::infra::git::redis: {}
+roles::infra::git::runner: {}
+roles::infra::git::server: {}
+roles::infra::halb::haproxy2: {}
+roles::infra::incus::imagehost: {}
+roles::infra::incus::node: {}
+roles::infra::k8s::compute: {}
+roles::infra::k8s::control: {}
+roles::infra::logs::vlagent: {}
+roles::infra::logs::vlinsert: {}
+roles::infra::logs::vlselect: {}
+roles::infra::logs::vlstorage: {}
+roles::infra::mail::backend: {}
+roles::infra::mail::gateway: {}
+roles::infra::metrics::grafana: {}
+roles::infra::metrics::prometheus: {}
+roles::infra::metrics::vmagent: {}
+roles::infra::metrics::vminsert: {}
+roles::infra::metrics::vmselect: {}
+roles::infra::metrics::vmstorage: {}
+roles::infra::nomad::agentv2: {}
+roles::infra::nomad::server: {}
+roles::infra::pki::certbot: {}
+roles::infra::proxy::jumphost: {}
+roles::infra::puppetboard::server: {}
+roles::infra::puppetdb::api: {}
+roles::infra::puppetdb::sql: {}
+roles::infra::puppet::master: {}
+roles::infra::reposync::repo: {}
+roles::infra::reposync::syncer: {}
+roles::infra::sql::shared: {}
+roles::infra::storage::consul: {}
+roles::infra::storage::edgecache: {}
+roles::infra::storage::vault: {}

config/encapi/statuses.yaml

@@ -0,0 +1,10 @@
+# encapi statuses == Puppet environments (Cobbler "status").
+# Seeded from what the estate actually runs (all nodes today report
+# catalog_environment=develop) plus production (the environment the incus
+# instance module pins via puppetca_certificate) and testing (implicit).
+production:
+  description: Production environment
+develop:
+  description: Development environment (current default across the estate)
+testing:
+  description: Implicit/transient environment

config/encapi/terragrunt.hcl

@@ -0,0 +1,19 @@
+locals {
+  statuses = yamldecode(file("${get_terragrunt_dir()}/statuses.yaml"))
+  roles    = yamldecode(file("${get_terragrunt_dir()}/roles.yaml"))
+  nodes    = yamldecode(file("${get_terragrunt_dir()}/nodes.yaml"))
+}
+
+include "root" {
+  path = find_in_parent_folders("root.hcl")
+}
+
+terraform {
+  source = "${get_repo_root()}/modules/encapi"
+}
+
+inputs = {
+  statuses = local.statuses
+  roles    = local.roles
+  nodes    = local.nodes
+}

config/root.hcl

@@ -26,7 +26,7 @@ inputs = {
 generate "backend" {
   path      = "backend.tf"
   if_exists = "overwrite_terragrunt"
-  contents = <<EOF
+  contents  = <<EOF
 terraform {
   required_providers {
     vault = {
@@ -49,6 +49,10 @@ terraform {
       source = "camptocamp/puppetdb"
       version = "2.0.0"
     }
+    encapi = {
+      source  = "git.unkin.net/unkin/encapi"
+      version = "0.1.0"
+    }
   }
   backend "consul" {
     address = "${local.consul_addr}"

modules/encapi/main.tf

@@ -0,0 +1,29 @@
+resource "encapi_status" "this" {
+  for_each = var.statuses
+
+  name        = each.key
+  description = each.value.description
+}
+
+resource "encapi_role" "this" {
+  for_each = var.roles
+
+  name           = each.key
+  description    = each.value.description
+  default_params = each.value.default_params == null ? null : jsonencode(each.value.default_params)
+}
+
+resource "encapi_node" "this" {
+  for_each = var.nodes
+
+  certname    = each.key
+  role        = each.value.role
+  environment = each.value.environment
+  params      = each.value.params == null ? null : jsonencode(each.value.params)
+
+  # Nodes FK-require their role and status to exist first.
+  depends_on = [
+    encapi_role.this,
+    encapi_status.this,
+  ]
+}

modules/encapi/providers.tf

@@ -0,0 +1,10 @@
+provider "encapi" {
+  endpoint = var.encapi_endpoint
+  # token defaults to the ENCAPI_WRITE_TOKEN environment variable
+}
+
+variable "encapi_endpoint" {
+  description = "The encapi server base URL."
+  type        = string
+  default     = "https://encapi.k8s.syd1.au.unkin.net"
+}

modules/encapi/variables.tf

@@ -0,0 +1,37 @@
+variable "statuses" {
+  description = <<EOT
+Map of encapi statuses (Puppet environments) to seed. Keyed by status name.
+Each value may carry an optional description.
+EOT
+  type = map(object({
+    description = optional(string)
+  }))
+  default = {}
+}
+
+variable "roles" {
+  description = <<EOT
+Map of encapi roles to seed. Keyed by role class name (e.g. roles::base).
+Each value may carry an optional description and a default_params object that
+is jsonencode()'d into the role's inheritable defaults.
+EOT
+  type = map(object({
+    description    = optional(string)
+    default_params = optional(any)
+  }))
+  default = {}
+}
+
+variable "nodes" {
+  description = <<EOT
+Map of encapi node assignments to seed. Keyed by certname. Each value pins the
+node to a role and environment (which must exist as a role/status above), with
+optional per-node params jsonencode()'d into the ENC output.
+EOT
+  type = map(object({
+    role        = string
+    environment = string
+    params      = optional(any)
+  }))
+  default = {}
+}

modules/encapi/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    encapi = {
+      source  = "git.unkin.net/unkin/encapi"
+      version = "0.1.0"
+    }
+  }
+}

modules/instance/main.tf

@@ -97,6 +97,17 @@ resource "puppetdb_node" "this" {
   depends_on = [incus_instance.this]
 }
 
+# Dual-write the ENC assignment into encapi alongside the Cobbler system above.
+# The role and environment must already exist in encapi (seeded by the
+# config/encapi leaf); apply that leaf before the instance leaves.
+resource "encapi_node" "this" {
+  certname    = "${var.name}.${var.cobbler_domain}"
+  role        = var.cobbler_mgmt_classes[0]
+  environment = var.encapi_environment
+
+  depends_on = [incus_instance.this]
+}
+
 resource "null_resource" "wait_for_instance_ready" {
   depends_on = [incus_instance.this]
 

modules/instance/providers.tf

@@ -17,3 +17,8 @@ provider "puppetca" {
   cert = var.puppet_cert_pub
   key  = var.puppet_cert_priv
 }
+
+provider "encapi" {
+  endpoint = var.encapi_endpoint
+  # token defaults to the ENCAPI_WRITE_TOKEN environment variable
+}

modules/instance/variables.tf

@@ -170,3 +170,18 @@ variable "check_on_instance_creation" {
   type        = bool
   default     = false
 }
+
+variable "encapi_endpoint" {
+  description = "The encapi server base URL."
+  type        = string
+  default     = "https://encapi.k8s.syd1.au.unkin.net"
+}
+
+variable "encapi_environment" {
+  description = <<EOT
+Environment (encapi_status) to pin this instance to in encapi. Defaults to
+"production" to match the environment used by puppetca_certificate today.
+EOT
+  type        = string
+  default     = "production"
+}