Trust the sshca host CA alongside the legacy signer (#530)

Catalog compilation moved to the k8s puppetserver compilers, which sign host certificates against the terraform-managed `sshca` mount. Clients only trust the legacy `ssh-host-signer` CA, so every re-signed node (ausyd1nxvm2120 already) presents a certificate nothing accepts, and knownhosts emits no plain host-key fallback.

- Add a second `@cert-authority *` entry for the `sshca` public key to `profiles::ssh::knownhosts::lines`.
- Keep the legacy entry untouched so legacy-signed hosts still verify.

Reviewed-on: #530
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

hieradata/common.yaml

@@ -367,6 +367,7 @@ ssh::server::options:
 
 profiles::ssh::knownhosts::lines:
   - '@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1HD97vYxLTniE4qNpGuftUlvmkEXIuX8+7nbENv/IzsGUghEDRtyThjQ7ojNKIsQ7f8wXr0gMcI+fAPfrbcOMHCAoYMomikwL0b3h95SZI40q3CyM+0DMnwiVVDX6C1QxkO2Rv9cszSkCa85NotJhXiUuTBI9BFcRPy+mAhbpAru+bfypYofI0wW97XNTl8Jgwmni5MgutBIQAokFIn5ux8iWxndCH3AqDtmkwC5DfQeQ+wZx7rkwqJEpJffQzrjb1gIM6P9hDCVBBVPh/3o80IJ69rFWrJAZUb+JpG4cXJH0NcSW+wqc3JCT/x3q8VlHwOTXSlNNKtOJCRx73mB8e1XTTy2a9FgpKDDg5XQXWHAViJDz1RTRL9gRefMylRgKz4bXoTuY9kJWM8hPTyUejtukbJThlBJc3OmDxBZBF7F0iqB11pHexok43OCEiANodVa36eWu9/5X032Vm48fZ1/akDPY/NSy3wAn7kwut+A0/JAHFHASrq+1mt9YurkJegI+YHXO6eEWpBIpmI7ORHJbGL4MhkHrxYzVamuP8CkU7tXzsv138+wpOcRHNp9yJY4PT40BZkRf/O3O+jt3pj9Dj8rvgywF2W6hFzywh3Y78upOprRkQlQtHfsI8EyrYI8/hUw2u3H+3yPXh3YjWfqvWVG1BRLRHBV7m90uaw=='
+  - '@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDi80G0GtKMdRj4azPwxbJW46NG0y8seSVSnvFm2Ka/nckdUb/3lRlQuPYf1tkbqqFlcXjDM8+u0tzM2kLsgfn0Dpujm1fuoA66yGGweBjtxLFErH5+6+/KND5I9w8LMY2AtVztnBk/CVx8RwgrooABDRZiBH7OOAOpJqqFI7LvEsv61zC0nAqbYJ8uxfx+r4lJ9dUhK1woitEqC4npSRUn5KJK+KAEd7AzcUkZb6TO9A3oRPz1nQ/qU+QNMmVUi+wRj9kJR18mxErugyLLTNcFDBqSdHNun3eUNBUmoS2cAa8ZVscOLzbUGejVK9UY06Q7wu+J34Fzl8CN5tBqRHGZ3ykqGZ6gG+O0Egr9Rm3Obgkujpio2uTh27LhBy72vjgJ8kTFmPlzANTJFpKlsy91usSFPh8WZeIo6VpPLqnC32rjfNkfqHyPAeJ3+rdOkUZoDjMoZc3wxxLZTgMU5ud1w4LxQNRkNiaT/tRRNQF8o+pygkS7xxKduBBMzYdRS1OifaS4gyvP82oOyquWywhyFNtG7ph8FQwc34puM7FyxfaJ0XL/+zAfPMhDoOojvofADJo73R+FgVyer+mCJw4KtWJ4JzZrNTYz1Xl8WlhF8RDzOYfSH46qzJFa9FcRqgP4LUkgzdvcQ+1hBFpvpHtw3vl3DiqtZDDbK9SyrEtdnw=='
 
 profiles::base::groups::local:
   admins:

hieradata/os/AlmaLinux/all_releases.yaml

@@ -77,6 +77,7 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'
   rpm-vendor:
     name: rpm-vendor
     descr: rpm-vendor repository
@@ -84,6 +85,7 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'
   # Per-release variants, resolved from the host's EL major version so el8
   # hosts pull rpm-internal-el8/rpm-vendor-el8, el9 hosts el9, etc.
   rpm-internal-release:
@@ -93,6 +95,7 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-el%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'
   rpm-vendor-release:
     name: rpm-vendor-el%{facts.os.release.major}
     descr: rpm-vendor-el%{facts.os.release.major} repository
@@ -100,6 +103,7 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-el%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'
 
   # Additional repositories - default to absent, roles can override with ensure: present
   # FRRouting repositories

hieradata/os/Fedora/all_releases.yaml

@@ -60,6 +60,7 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal-f%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'
   rpm-vendor:
     name: rpm-vendor-f%{facts.os.release.major}
     descr: rpm-vendor-f%{facts.os.release.major} repository
@@ -67,3 +68,4 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-vendor-f%{facts.os.release.major}/
     gpgcheck: false
     mirrorlist: absent
+    metadata_expire: '60'

hieradata/roles/infra/puppet/master.yaml

@@ -26,6 +26,15 @@ profiles::puppet::cobbler_enc::packages:
   - 'requests'
   - 'PyYAML'
 profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-enc.git
+
+# Deep-merged with the entries in roles/infra/puppet.yaml.
+profiles::packages::include:
+  encapic:
+    ensure: '0.2.0'
+
+profiles::puppet::encapic::encapi_url: https://encapi.k8s.syd1.au.unkin.net
+profiles::puppet::server::external_nodes: '/usr/bin/encapic-enc'
+
 profiles::puppet::r10k::r10k_repo: https://git.unkin.net/unkin/puppet-r10k.git
 profiles::puppet::g10k::bin_path: '/usr/bin/g10k'
 profiles::puppet::g10k::cfg_path: '/etc/puppetlabs/r10k/r10k.yaml'

modules/libs/lib/facter/enc_direct_facts.rb

@@ -3,13 +3,21 @@
 require 'facter'
 require 'yaml'
 require 'net/http'
+require 'openssl'
 require 'uri'
 require 'fileutils'
 
-# CobblerENC module: Fetches ENC data from Cobbler, caches it, and provides structured facts.
+# EncapiENC module: Fetches ENC data from encapi, caches it, and provides structured facts.
-module CobblerENC
+module EncapiENC
   CACHE_FILE = '/var/cache/puppet_enc.yaml'
   CACHE_TTL = 7 * 24 * 60 * 60 # 7 days in seconds
+  # Facter runs under Puppet's vendored ruby, whose OpenSSL trusts only
+  # /opt/puppetlabs/puppet/ssl/cert.pem and never the system trust store, so the
+  # estate CA anchor profiles::pki::vaultca installs has to be named explicitly.
+  CA_BUNDLE_PATHS = [
+    '/etc/pki/ca-trust/source/anchors/vaultcaroot.pem',
+    '/usr/local/share/ca-certificates/vaultcaroot.pem'
+  ].freeze
   @enc_data = nil # In-memory cache for the ENC response
 
   def self.read_cache
@@ -29,9 +37,22 @@ module CobblerENC
     File.write(CACHE_FILE, cache_data.to_yaml)
   end
 
-  def self.fetch_from_cobbler
+  def self.ca_bundle
-    uri = URI("http://cobbler.main.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}")
+    CA_BUNDLE_PATHS.find { |path| File.exist?(path) }
-    response = Net::HTTP.get_response(uri)
+  end
+
+  def self.http_client(uri)
+    client = Net::HTTP.new(uri.host, uri.port)
+    client.use_ssl = true
+    client.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    bundle = ca_bundle
+    client.ca_file = bundle if bundle
+    client
+  end
+
+  def self.fetch_from_encapi
+    uri = URI("https://encapi.k8s.syd1.au.unkin.net/cblr/svc/op/puppet/hostname/#{Facter.value(:fqdn) || Facter.value(:hostname)}")
+    response = http_client(uri).request(Net::HTTP::Get.new(uri))
 
     raise "Failed to fetch ENC data. HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
 
@@ -41,7 +62,7 @@ module CobblerENC
   def self.retrieve_enc_data
     return @enc_data if @enc_data
 
-    @enc_data = fetch_from_cobbler
+    @enc_data = fetch_from_encapi
     write_cache(@enc_data)
     @enc_data
   end
@@ -49,26 +70,26 @@ module CobblerENC
   def self.fetch_enc_data
     retrieve_enc_data
   rescue StandardError => e
-    Facter.warn("Error retrieving Cobbler ENC data: #{e.message}")
+    Facter.warn("Error retrieving encapi ENC data: #{e.message}")
     @enc_data = read_cache
     return @enc_data unless @enc_data.empty?
 
-    raise 'No cached ENC data available and Cobbler is down.'
+    raise 'No cached ENC data available and encapi is unreachable.'
   end
 
   def self.enc_role
-    fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in Cobbler ENC response')
+    fetch_enc_data.fetch('classes', {}).keys.first || raise('ENC Role not found in encapi ENC response')
   end
 
   def self.enc_env
-    fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in Cobbler ENC response')
+    fetch_enc_data.fetch('environment', nil) || raise('ENC Environment not found in encapi ENC response')
   end
 end
 
 Facter.add('enc_role') do
-  setcode { CobblerENC.enc_role }
+  setcode { EncapiENC.enc_role }
 end
 
 Facter.add('enc_env') do
-  setcode { CobblerENC.enc_env }
+  setcode { EncapiENC.enc_env }
 end

site/profiles/manifests/dns/updater.pp

@@ -24,8 +24,7 @@ class profiles::dns::updater (
   Stdlib::AbsolutePath        $config_dir      = '/etc/dns-updater',
   Stdlib::AbsolutePath        $master_basedir  = lookup('profiles::dns::master::basedir'),
   # dns-updater daemon (replaces the dns-update shell script). 'latest' so hosts
-  # pick up new releases (e.g. the record filter); rpm-internal metadata_expire
+  # pick up new releases (e.g. the record filter).
-  # is 1h so this does not thrash.
   String                      $package_ensure   = 'latest',
   Stdlib::AbsolutePath        $api_socket       = '/run/dns-updater/api.sock',
   String                      $resync           = '10m',

site/profiles/manifests/puppet/encapic.pp

@@ -0,0 +1,32 @@
+# Class: profiles::puppet::encapic
+#
+# Manages the configuration for the encapic ENC client. The package itself is
+# installed through profiles::packages (pinned in hiera); this class owns the
+# config so the encapi endpoint can change without repackaging.
+class profiles::puppet::encapic (
+  Stdlib::HTTPUrl       $encapi_url,
+  Stdlib::AbsolutePath  $config_dir = '/etc/encapic',
+  String                $config_name = 'encapic.conf',
+  String                $owner = 'root',
+  String                $group = 'root',
+) {
+
+  # The RPM ships this file as %config(noreplace), so puppet must write it only
+  # once the package is present or the install overwrites it.
+  file { $config_dir:
+    ensure  => directory,
+    mode    => '0755',
+    owner   => $owner,
+    group   => $group,
+    require => Package['encapic'],
+  }
+
+  file { "${config_dir}/${config_name}":
+    ensure  => file,
+    mode    => '0644',
+    owner   => $owner,
+    group   => $group,
+    content => "ENCAPI_URL=${encapi_url}\n",
+    require => File[$config_dir],
+  }
+}

site/profiles/manifests/puppet/puppetmaster.pp

@@ -12,6 +12,7 @@ class profiles::puppet::puppetmaster (
     include profiles::puppet::g10k
     include profiles::puppet::enc
     include profiles::puppet::cobbler_enc
+    include profiles::puppet::encapic
     include profiles::puppet::autosign
     include profiles::puppet::gems
     include profiles::helpers::certmanager