Let certmanager and sshsignhost authenticate to Vault by kubernetes auth

Both helpers are run server-side by generate() during catalog compilation and
only speak AppRole, whose token_bound_cidrs pin them to the six legacy VM
masters, so the autoscaled k8s compilers cannot obtain a token and any compile
needing a cert or a signed host key fails there.

- Add a kubernetes login branch that reads the service account JWT and posts it
  to auth/<k8s_mount>/login, selected by an auth_method config key
- Add auth_method, k8s_mount, k8s_role and jwt_path class parameters, defaulting
  to approle so the VM masters render and behave as before
- Render role_id and approle_path only for the approle case
- Report a missing JWT, a rejected login or an unknown auth_method on stderr
  instead of falling back or raising
- Point sshsignhost at the sshca mount and signhost role that Vault actually has

Needs terraform-vault #152, already applied.

hieradata/os/AlmaLinux/all_releases.yaml

@@ -77,7 +77,6 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'
   rpm-vendor:
     name: rpm-vendor
     descr: rpm-vendor repository
@@ -85,7 +84,6 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'
   # Per-release variants, resolved from the host's EL major version so el8
   # hosts pull rpm-internal-el8/rpm-vendor-el8, el9 hosts el9, etc.
   rpm-internal-release:
@@ -95,7 +93,6 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-el%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'
   rpm-vendor-release:
     name: rpm-vendor-el%{facts.os.release.major}
     descr: rpm-vendor-el%{facts.os.release.major} repository
@@ -103,7 +100,6 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-el%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'
 
   # Additional repositories - default to absent, roles can override with ensure: present
   # FRRouting repositories

hieradata/os/Fedora/all_releases.yaml

@@ -60,7 +60,6 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-f%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'
   rpm-vendor:
     name: rpm-vendor-f%{facts.os.release.major}
     descr: rpm-vendor-f%{facts.os.release.major} repository
@@ -68,4 +67,3 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-f%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
-    metadata_expire: '60'

hieradata/roles/infra/puppet/master.yaml

@@ -26,15 +26,6 @@ profiles::puppet::cobbler_enc::packages:
   - 'requests'
   - 'PyYAML'
 profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-enc.git
-
-# Deep-merged with the entries in roles/infra/puppet.yaml.
-profiles::packages::include:
-  encapic:
-    ensure: '0.2.0'
-
-profiles::puppet::encapic::encapi_url: https://encapi.k8s.syd1.au.unkin.net
-profiles::puppet::server::external_nodes: '/usr/bin/encapic-enc'
-
 profiles::puppet::r10k::r10k_repo: https://git.unkin.net/unkin/puppet-r10k.git
 profiles::puppet::g10k::bin_path: '/usr/bin/g10k'
 profiles::puppet::g10k::cfg_path: '/etc/puppetlabs/r10k/r10k.yaml'
@@ -56,9 +47,9 @@ profiles::helpers::certmanager::vault_config:
 
 profiles::helpers::sshsignhost::vault_config:
   addr: 'https://vault.service.consul:8200'
-  mount_point: 'ssh-host-signer'
+  mount_point: 'sshca'
   approle_path: 'approle'
-  role_name: 'hostrole'
+  role_name: 'signhost'
   output_path: '/tmp/sshsignhost'
   role_id: "%{lookup('sshsignhost::role_id')}"
 

modules/libs/lib/facter/enc_direct_facts.rb

@@ -3,21 +3,13 @@
 require 'facter'
 require 'yaml'
 require 'net/http'
-require 'openssl'
 require 'uri'
 require 'fileutils'
 
-# EncapiENC module: Fetches ENC data from encapi, caches it, and provides structured facts.
+# CobblerENC module: Fetches ENC data from Cobbler, caches it, and provides structured facts.
-module EncapiENC
+module CobblerENC
   CACHE_FILE = '/var/cache/puppet_enc.yaml'
   CACHE_TTL = 7 * 24 * 60 * 60 # 7 days in seconds
-  # Facter runs under Puppet's vendored ruby, whose OpenSSL trusts only
-  # /opt/puppetlabs/puppet/ssl/cert.pem and never the system trust store, so the
-  # estate CA anchor profiles::pki::vaultca installs has to be named explicitly.
-  CA_BUNDLE_PATHS = [
-    '/etc/pki/ca-trust/source/anchors/vaultcaroot.pem',
-    '/usr/local/share/ca-certificates/vaultcaroot.pem'
-  ].freeze
   @enc_data = nil # In-memory cache for the ENC response
 
   def self.read_cache
@@ -37,22 +29,9 @@ module EncapiENC
     File.write(CACHE_FILE, cache_data.to_yaml)
   end
 
-  def self.ca_bundle
+  def self.fetch_from_cobbler
-    CA_BUNDLE_PATHS.find { |path| File.exist?(path) }
+    uri = URI("http://cobbler.main.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}")
-  end
+    response = Net::HTTP.get_response(uri)
-
-  def self.http_client(uri)
-    client = Net::HTTP.new(uri.host, uri.port)
-    client.use_ssl = true
-    client.verify_mode = OpenSSL::SSL::VERIFY_PEER
-    bundle = ca_bundle
-    client.ca_file = bundle if bundle
-    client
-  end
-
-  def self.fetch_from_encapi
-    uri = URI("https://encapi.k8s.syd1.au.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}")
-    response = http_client(uri).request(Net::HTTP::Get.new(uri))
 
     raise "Failed to fetch ENC data. HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
 
@@ -62,7 +41,7 @@ module EncapiENC
   def self.retrieve_enc_data
     return @enc_data if @enc_data
 
-    @enc_data = fetch_from_encapi
+    @enc_data = fetch_from_cobbler
     write_cache(@enc_data)
     @enc_data
   end
@@ -70,26 +49,26 @@ module EncapiENC
   def self.fetch_enc_data
     retrieve_enc_data
   rescue StandardError => e
-    Facter.warn("Error retrieving encapi ENC data: #{e.message}")
+    Facter.warn("Error retrieving Cobbler ENC data: #{e.message}")
     @enc_data = read_cache
     return @enc_data unless @enc_data.empty?
 
-    raise 'No cached ENC data available and encapi is unreachable.'
+    raise 'No cached ENC data available and Cobbler is down.'
   end
 
   def self.enc_role
-    fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in encapi ENC response')
+    fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in Cobbler ENC response')
   end
 
   def self.enc_env
-    fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in encapi ENC response')
+    fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in Cobbler ENC response')
   end
 end
 
 Facter.add('enc_role') do
-  setcode { EncapiENC.enc_role }
+  setcode { CobblerENC.enc_role }
 end
 
 Facter.add('enc_env') do
-  setcode { EncapiENC.enc_env }
+  setcode { CobblerENC.enc_env }
 end

site/profiles/manifests/dns/updater.pp

@@ -24,7 +24,8 @@ class profiles::dns::updater (
   Stdlib::AbsolutePath        $config_dir      = '/etc/dns-updater',
   Stdlib::AbsolutePath        $master_basedir  = lookup('profiles::dns::master::basedir'),
   # dns-updater daemon (replaces the dns-update shell script). 'latest' so hosts
-  # pick up new releases (e.g. the record filter).
+  # pick up new releases (e.g. the record filter); rpm-internal metadata_expire
+  # is 1h so this does not thrash.
   String                      $package_ensure   = 'latest',
   Stdlib::AbsolutePath        $api_socket       = '/run/dns-updater/api.sock',
   String                      $resync           = '10m',

site/profiles/manifests/helpers/certmanager.pp

@@ -7,6 +7,10 @@ class profiles::helpers::certmanager (
   Stdlib::AbsolutePath  $venv_path = "${base_path}/venv",
   Stdlib::AbsolutePath  $config_path = "${base_path}/config.yaml",
   Hash                  $vault_config = {},
+  Enum['approle','kubernetes'] $auth_method = 'approle',
+  String[1]             $k8s_mount = 'k8s/au/syd1',
+  String[1]             $k8s_role = 'puppet_certmanager',
+  Stdlib::AbsolutePath  $jwt_path = '/var/run/secrets/kubernetes.io/serviceaccount/token',
   String                $owner = 'root',
   String                $group = 'root',
   Boolean               $systempkgs = false,
@@ -16,6 +20,14 @@ class profiles::helpers::certmanager (
 
   if $::facts['python3_version'] {
 
+    # class parameters supply the auth defaults; $vault_config may override them
+    $vault_settings = {
+      'auth_method' => $auth_method,
+      'k8s_mount'   => $k8s_mount,
+      'k8s_role'    => $k8s_role,
+      'jwt_path'    => $jwt_path,
+    } + $vault_config
+
     $python_version = $version ? {
       'system' => $::facts['python3_version'],
       default  => $version,

site/profiles/manifests/helpers/sshsignhost.pp

@@ -7,6 +7,10 @@ class profiles::helpers::sshsignhost (
   Stdlib::AbsolutePath  $venv_path = "${base_path}/venv",
   Stdlib::AbsolutePath  $config_path = "${base_path}/config.yaml",
   Hash                  $vault_config = {},
+  Enum['approle','kubernetes'] $auth_method = 'approle',
+  String[1]             $k8s_mount = 'k8s/au/syd1',
+  String[1]             $k8s_role = 'puppet_sshsigner',
+  Stdlib::AbsolutePath  $jwt_path = '/var/run/secrets/kubernetes.io/serviceaccount/token',
   String                $owner = 'root',
   String                $group = 'root',
   Boolean               $systempkgs = false,
@@ -16,6 +20,14 @@ class profiles::helpers::sshsignhost (
 
   if $::facts['python3_version'] {
 
+    # class parameters supply the auth defaults; $vault_config may override them
+    $vault_settings = {
+      'auth_method' => $auth_method,
+      'k8s_mount'   => $k8s_mount,
+      'k8s_role'    => $k8s_role,
+      'jwt_path'    => $jwt_path,
+    } + $vault_config
+
     $python_version = $version ? {
       'system' => $::facts['python3_version'],
       default  => $version,

site/profiles/manifests/puppet/encapic.pp

@@ -1,32 +0,0 @@
-# Class: profiles::puppet::encapic
-#
-# Manages the configuration for the encapic ENC client. The package itself is
-# installed through profiles::packages (pinned in hiera); this class owns the
-# config so the encapi endpoint can change without repackaging.
-class profiles::puppet::encapic (
-  Stdlib::HTTPUrl       $encapi_url,
-  Stdlib::AbsolutePath  $config_dir = '/etc/encapic',
-  String                $config_name = 'encapic.conf',
-  String                $owner = 'root',
-  String                $group = 'root',
-) {
-
-  # The RPM ships this file as %config(noreplace), so puppet must write it only
-  # once the package is present or the install overwrites it.
-  file { $config_dir:
-    ensure  => directory,
-    mode    => '0755',
-    owner   => $owner,
-    group   => $group,
-    require => Package['encapic'],
-  }
-
-  file { "${config_dir}/${config_name}":
-    ensure  => file,
-    mode    => '0644',
-    owner   => $owner,
-    group   => $group,
-    content => "ENCAPI_URL=${encapi_url}\n",
-    require => File[$config_dir],
-  }
-}

site/profiles/manifests/puppet/puppetmaster.pp

@@ -12,7 +12,6 @@ class profiles::puppet::puppetmaster (
     include profiles::puppet::g10k
     include profiles::puppet::enc
     include profiles::puppet::cobbler_enc
-    include profiles::puppet::encapic
     include profiles::puppet::autosign
     include profiles::puppet::gems
     include profiles::helpers::certmanager

site/profiles/templates/helpers/certmanager.erb

@@ -1,6 +1,7 @@
 #!<%= @venv_path %>/bin/python
 
 import argparse
+import sys
 import requests
 import json
 import os
@@ -25,14 +26,50 @@ def authenticate_approle(vault_config):
         auth_response = response.json()
         return auth_response['auth']['client_token']
     else:
-        print(f"Error authenticating with AppRole: {response.text}")
+        print(f"Error authenticating with AppRole: {response.text}", file=sys.stderr)
         return None
 
-def request_certificate(common_name, alt_names, ip_sans, expiry_days, vault_config):
+class VaultAuthError(Exception):
-    # Authenticate using AppRole and get a token
+    pass
+
+def authenticate_kubernetes(vault_config):
+    jwt_path = vault_config.get('jwt_path') or '/var/run/secrets/kubernetes.io/serviceaccount/token'
+    try:
+        with open(jwt_path, 'r') as file:
+            jwt = file.read().strip()
+    except OSError as error:
+        raise VaultAuthError(f"cannot read service account token '{jwt_path}': {error}")
+    if not jwt:
+        raise VaultAuthError(f"service account token '{jwt_path}' is empty")
+    url = f"{vault_config['addr']}/v1/auth/{vault_config['k8s_mount']}/login"
+    payload = {
+        "role": vault_config['k8s_role'],
+        "jwt": jwt,
+    }
+    response = requests.post(url, json=payload, verify=False)
+    if response.status_code != 200:
+        raise VaultAuthError(
+            f"kubernetes login as role '{vault_config['k8s_role']}' on mount "
+            f"'{vault_config['k8s_mount']}' was rejected ({response.status_code}): {response.text}"
+        )
+    return response.json()['auth']['client_token']
+
+def authenticate(vault_config):
+    auth_method = vault_config.get('auth_method', 'approle')
+    if auth_method == 'approle':
         client_token = authenticate_approle(vault_config)
         if not client_token:
-        print("Failed to authenticate with Vault using AppRole.")
+            raise VaultAuthError("approle login was rejected")
+        return client_token
+    if auth_method == 'kubernetes':
+        return authenticate_kubernetes(vault_config)
+    raise VaultAuthError(f"unsupported auth_method '{auth_method}': expected 'approle' or 'kubernetes'")
+
+def request_certificate(common_name, alt_names, ip_sans, expiry_days, vault_config):
+    try:
+        client_token = authenticate(vault_config)
+    except VaultAuthError as error:
+        print(f"Failed to authenticate with Vault: {error}", file=sys.stderr)
         return None
 
     url = f"{vault_config['addr']}/v1/{vault_config['mount_point']}/issue/{vault_config['role_name']}"
@@ -47,7 +84,7 @@ def request_certificate(common_name, alt_names, ip_sans, expiry_days, vault_conf
     if response.status_code == 200:
         return response.json()
     else:
-        print(f"Error requesting certificate: {response.text}")
+        print(f"Error requesting certificate: {response.text}", file=sys.stderr)
         return None
 
 def save_cert_files(certificate_response, common_name, compress, config, json_output):
@@ -95,7 +132,8 @@ def main(config_file):
         else:
             save_cert_files(certificate_response, args.common_name, args.compress, config, False)
     else:
-        print("Failed to obtain certificate.")
+        print("Failed to obtain certificate.", file=sys.stderr)
+        exit(1)
 
 if __name__ == "__main__":
     config_file = '<%= @config_path %>'

site/profiles/templates/helpers/certmanager_config.yaml.erb

@@ -1,7 +1,14 @@
 vault:
-  addr: '<%= @vault_config['addr'] %>'
+  addr: '<%= @vault_settings['addr'] %>'
-  role_id: '<%= @vault_config['role_id'] %>'
+  auth_method: '<%= @vault_settings['auth_method'] %>'
-  approle_path: '<%= @vault_config['approle_path'] %>'
+<% if @vault_settings['auth_method'] == 'kubernetes' -%>
-  mount_point: '<%= @vault_config['mount_point'] %>'
+  k8s_mount: '<%= @vault_settings['k8s_mount'] %>'
-  role_name: '<%= @vault_config['role_name'] %>'
+  k8s_role: '<%= @vault_settings['k8s_role'] %>'
-output_path: '<%= @vault_config['output_path'] %>'
+  jwt_path: '<%= @vault_settings['jwt_path'] %>'
+<% else -%>
+  role_id: '<%= @vault_settings['role_id'] %>'
+  approle_path: '<%= @vault_settings['approle_path'] %>'
+<% end -%>
+  mount_point: '<%= @vault_settings['mount_point'] %>'
+  role_name: '<%= @vault_settings['role_name'] %>'
+output_path: '<%= @vault_settings['output_path'] %>'

site/profiles/templates/helpers/sshsignhost.erb

@@ -1,5 +1,6 @@
 #!<%= @venv_path %>/bin/python
 import argparse
+import sys
 import requests
 import json
 import yaml
@@ -22,14 +23,50 @@ def authenticate_approle(vault_config):
         auth_response = response.json()
         return auth_response['auth']['client_token']
     else:
-        print(f"Error authenticating with AppRole: {response.text}")
+        print(f"Error authenticating with AppRole: {response.text}", file=sys.stderr)
         return None
 
-def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
+class VaultAuthError(Exception):
-    # Authenticate using AppRole and get a token
+    pass
+
+def authenticate_kubernetes(vault_config):
+    jwt_path = vault_config.get('jwt_path') or '/var/run/secrets/kubernetes.io/serviceaccount/token'
+    try:
+        with open(jwt_path, 'r') as file:
+            jwt = file.read().strip()
+    except OSError as error:
+        raise VaultAuthError(f"cannot read service account token '{jwt_path}': {error}")
+    if not jwt:
+        raise VaultAuthError(f"service account token '{jwt_path}' is empty")
+    url = f"{vault_config['addr']}/v1/auth/{vault_config['k8s_mount']}/login"
+    payload = {
+        "role": vault_config['k8s_role'],
+        "jwt": jwt,
+    }
+    response = requests.post(url, json=payload, verify=False)
+    if response.status_code != 200:
+        raise VaultAuthError(
+            f"kubernetes login as role '{vault_config['k8s_role']}' on mount "
+            f"'{vault_config['k8s_mount']}' was rejected ({response.status_code}): {response.text}"
+        )
+    return response.json()['auth']['client_token']
+
+def authenticate(vault_config):
+    auth_method = vault_config.get('auth_method', 'approle')
+    if auth_method == 'approle':
         client_token = authenticate_approle(vault_config)
         if not client_token:
-        print("Failed to authenticate with Vault using AppRole.")
+            raise VaultAuthError("approle login was rejected")
+        return client_token
+    if auth_method == 'kubernetes':
+        return authenticate_kubernetes(vault_config)
+    raise VaultAuthError(f"unsupported auth_method '{auth_method}': expected 'approle' or 'kubernetes'")
+
+def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
+    try:
+        client_token = authenticate(vault_config)
+    except VaultAuthError as error:
+        print(f"Failed to authenticate with Vault: {error}", file=sys.stderr)
         return None
 
     # Prepare the SSH certificate signing request
@@ -47,7 +84,7 @@ def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
     if response.status_code == 200:
         return response.json()
     else:
-        print(f"Error requesting certificate: {response.text}")
+        print(f"Error requesting certificate: {response.text}", file=sys.stderr)
         return None
 
 def main(config_file):
@@ -75,7 +112,7 @@ def main(config_file):
         else:
             print(response['data']['signed_key'])
     else:
-        print("Error: The response does not contain the expected data.")
+        print("Error: The response does not contain the expected data.", file=sys.stderr)
         exit(1)
 
 if __name__ == "__main__":

site/profiles/templates/helpers/sshsignhost_config.yaml.erb

@@ -1,7 +1,14 @@
 vault:
-  addr: '<%= @vault_config['addr'] %>'
+  addr: '<%= @vault_settings['addr'] %>'
-  role_id: '<%= @vault_config['role_id'] %>'
+  auth_method: '<%= @vault_settings['auth_method'] %>'
-  approle_path: '<%= @vault_config['approle_path'] %>'
+<% if @vault_settings['auth_method'] == 'kubernetes' -%>
-  mount_point: '<%= @vault_config['mount_point'] %>'
+  k8s_mount: '<%= @vault_settings['k8s_mount'] %>'
-  role_name: '<%= @vault_config['role_name'] %>'
+  k8s_role: '<%= @vault_settings['k8s_role'] %>'
-output_path: '<%= @vault_config['output_path'] %>'
+  jwt_path: '<%= @vault_settings['jwt_path'] %>'
+<% else -%>
+  role_id: '<%= @vault_settings['role_id'] %>'
+  approle_path: '<%= @vault_settings['approle_path'] %>'
+<% end -%>
+  mount_point: '<%= @vault_settings['mount_point'] %>'
+  role_name: '<%= @vault_settings['role_name'] %>'
+output_path: '<%= @vault_settings['output_path'] %>'