Merge pull request 'neoloc/sshsign_hostkeys' (#23) from neoloc/sshsign_hostkeys into develop

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/23

Puppetfile

@@ -42,6 +42,7 @@ mod 'puppet-filemapper', '4.0.0'
 # other
 mod 'ghoneycutt-puppet', '3.3.0'
 mod 'saz-sudo', '8.0.0'
+mod 'saz-ssh', '12.1.0'
 mod 'ghoneycutt-timezone', '4.0.0'
 mod 'dalen-puppetdbquery', '3.0.1'
 mod 'markt-galera', '3.1.0'

hieradata/common.yaml

@@ -120,6 +120,7 @@ facts_path: '/opt/puppetlabs/facter/facts.d'
 hiera_include:
   - timezone
   - networking
+  - ssh::server
 
 profiles::ntp::client::ntp_role: 'roles::infra::ntp::server'
 profiles::ntp::client::use_ntp: 'region'
@@ -222,6 +223,38 @@ puppetdbsql: puppetdbsql.service.au-syd1.consul
 prometheus::node_exporter::export_scrape_job: true
 prometheus::systemd_exporter::export_scrape_job: true
 
+ssh::server::storeconfigs_enabled: false
+ssh::server::options:
+  Protocol: '2'
+  ListenAddress:
+    - '127.0.0.1'
+    - '%{facts.networking.ip}'
+  SyslogFacility: 'AUTHPRIV'
+  HostKey:
+    - /etc/ssh/ssh_host_rsa_key
+    - /etc/ssh/ssh_host_ecdsa_key
+    - /etc/ssh/ssh_host_ed25519_key
+  HostCertificate: /etc/ssh/ssh_host_rsa_key-cert.pem
+  AuthorizedKeysFile: .ssh/authorized_keys
+  PermitRootLogin: no
+  PasswordAuthentication: no
+  ChallengeResponseAuthentication: no
+  PubkeyAuthentication: yes
+  GSSAPIAuthentication: yes
+  GSSAPICleanupCredentials: yes
+  UsePAM: yes
+  X11Forwarding: no
+  PrintMotd: no
+  AcceptEnv:
+    - LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
+    - LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
+    - LC_IDENTIFICATION LC_ALL LANGUAGE
+    - XMODIFIERS
+  Subsystem: sftp /usr/libexec/openssh/sftp-server
+
+profiles::ssh::knownhosts::lines:
+  - '@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1HD97vYxLTniE4qNpGuftUlvmkEXIuX8+7nbENv/IzsGUghEDRtyThjQ7ojNKIsQ7f8wXr0gMcI+fAPfrbcOMHCAoYMomikwL0b3h95SZI40q3CyM+0DMnwiVVDX6C1QxkO2Rv9cszSkCa85NotJhXiUuTBI9BFcRPy+mAhbpAru+bfypYofI0wW97XNTl8Jgwmni5MgutBIQAokFIn5ux8iWxndCH3AqDtmkwC5DfQeQ+wZx7rkwqJEpJffQzrjb1gIM6P9hDCVBBVPh/3o80IJ69rFWrJAZUb+JpG4cXJH0NcSW+wqc3JCT/x3q8VlHwOTXSlNNKtOJCRx73mB8e1XTTy2a9FgpKDDg5XQXWHAViJDz1RTRL9gRefMylRgKz4bXoTuY9kJWM8hPTyUejtukbJThlBJc3OmDxBZBF7F0iqB11pHexok43OCEiANodVa36eWu9/5X032Vm48fZ1/akDPY/NSy3wAn7kwut+A0/JAHFHASrq+1mt9YurkJegI+YHXO6eEWpBIpmI7ORHJbGL4MhkHrxYzVamuP8CkU7tXzsv138+wpOcRHNp9yJY4PT40BZkRf/O3O+jt3pj9Dj8rvgywF2W6hFzywh3Y78upOprRkQlQtHfsI8EyrYI8/hUw2u3H+3yPXh3YjWfqvWVG1BRLRHBV7m90uaw=='
+
 profiles::base::groups::local:
   admins:
     ensure: present

hieradata/country/au/region/drw1/infra/puppet/master.eyaml

@@ -1,3 +1,4 @@
 ---
 certmanager::vault_token: ENC[PKCS7,MIIBygYJKoZIhvcNAQcDoIIBuzCCAbcCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAWh7bsttz/JCBo/CPoCgA2doo3jO6jT6NsOoE3/06W2IW+Ij6KHKYILMkG3tS4NAegMI48QR9n++4Xa7u+97w1HO4ENpfLrkuKUcWUFCxxb2OdWhxucIlt3Ay/2+tofOSvqiRKeEISBtOK//Q1a4Iu5GwEP+lvDQ5rcoS0dryNie/okXaLratWOsmctJ6LFuUw5siCcFyUzfvr2ROsB14YoF989np+X1dJqBWxcLmbVNKx766GrRhb1WGeF0qxounCmWEKGt0zY4Zk27KNFlFu7XByDWZoSCVCMvkQaRKhvdNA39Y9vscZJGPGFhz+qKPoeqwUidz0IY51CaFSXewmzCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQC+e2iOlFLlr9inVU8nEVWIBgqb0u/ICsLtxZqOpN9OIFWl+4hVrvTo24JzTc1jMSCONeL4Ab7jtTMbsweE9zUf6XlwhHLXfxfg7FL3WBsOWCUBXIAh338cZCXUGX7m0Qvtgg3VTEbTNDJhZle8Sjo6Gl]
 certmanager::role_id: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAJuE+uzgQaBRUXBCigckEo1j+UxxbiUGrzdf/B9K7XPdVxZh6TzYLpBgNnyaT6vLo0boX4uRD/By0gT5R/2qcXD6d/j+fh517Ctk4d2uO64f0vH3PzyyOBalsNtcCdPiV3q/xGqzQSHhPiNkFEjDvMBz5p53UjfKA6gAiPrLklp4rN/NVyiLBw20NeIqbL25VdkQa13ViS0Gm/eUQu7a2xQ1dvQFWWfuLaQxO0dh8L0ynkfmWKIjaiD5412Z8hYURu0otxbqVDdIbEMx5xQsXnFKeN93yHmgs7a7M6fLdp9jh+G8B+IlK1W7/9v2+RT0/yI3ZgWHVTvDRhMHuPGBjfTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC5avtOPp9N65U1ILQENnvAgDBqI8XAjqbWIvXHqOEiKYdu+co0EEtsHR1v5xAeCmj/ZA6MLeKFlAVJbvpyCpzjons=]
+sshsignhost::role_id: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAT86C/InXrgDtXCc9NFze91YMvjTNDqWgv4uzPFI48clOeQyD6x+vOHWP2yNp1OyNHcYLCiLyrv+rSIQyXlLnbeyZWV+7kXIon057Tp7l0BxWtd0hjQEcyWzqQQE7R264C8/qKRak81LIu6RshWZAchYo/BMPuOqVr0m+1zDwOV9JwZc3bpexzsl57CK5pesOrpfdvnd/xrOoEMR+P+C5PC6QLtQl3zkOD3N9kP6HqwbhWH5ZBPy88Kc+5kYM6QVpQSjFIIHK1SWsN0VZoxpkuFlFXB5KHDgZtg3kxrofzjQghl41zJBCDq9Z5oZ+2b1p/j/9jCASyp/ju68H5WXzbzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCf4Nqp6SAl/XjmhPDnTvVJgDCdDhxWaChhjJ3eRcW4NTFgf3zm7Bu65za0li26FKuKks00duF4zebfNw7ZUVsYtIU=]

hieradata/country/au/region/syd1/infra/puppet/master.eyaml

@@ -1,3 +1,4 @@
 ---
 certmanager::vault_token: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAJuE+uzgQaBRUXBCigckEo1j+UxxbiUGrzdf/B9K7XPdVxZh6TzYLpBgNnyaT6vLo0boX4uRD/By0gT5R/2qcXD6d/j+fh517Ctk4d2uO64f0vH3PzyyOBalsNtcCdPiV3q/xGqzQSHhPiNkFEjDvMBz5p53UjfKA6gAiPrLklp4rN/NVyiLBw20NeIqbL25VdkQa13ViS0Gm/eUQu7a2xQ1dvQFWWfuLaQxO0dh8L0ynkfmWKIjaiD5412Z8hYURu0otxbqVDdIbEMx5xQsXnFKeN93yHmgs7a7M6fLdp9jh+G8B+IlK1W7/9v2+RT0/yI3ZgWHVTvDRhMHuPGBjfTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC5avtOPp9N65U1ILQENnvAgDBqI8XAjqbWIvXHqOEiKYdu+co0EEtsHR1v5xAeCmj/ZA6MLeKFlAVJbvpyCpzjons=]
 certmanager::role_id: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAJuE+uzgQaBRUXBCigckEo1j+UxxbiUGrzdf/B9K7XPdVxZh6TzYLpBgNnyaT6vLo0boX4uRD/By0gT5R/2qcXD6d/j+fh517Ctk4d2uO64f0vH3PzyyOBalsNtcCdPiV3q/xGqzQSHhPiNkFEjDvMBz5p53UjfKA6gAiPrLklp4rN/NVyiLBw20NeIqbL25VdkQa13ViS0Gm/eUQu7a2xQ1dvQFWWfuLaQxO0dh8L0ynkfmWKIjaiD5412Z8hYURu0otxbqVDdIbEMx5xQsXnFKeN93yHmgs7a7M6fLdp9jh+G8B+IlK1W7/9v2+RT0/yI3ZgWHVTvDRhMHuPGBjfTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC5avtOPp9N65U1ILQENnvAgDBqI8XAjqbWIvXHqOEiKYdu+co0EEtsHR1v5xAeCmj/ZA6MLeKFlAVJbvpyCpzjons=]
+sshsignhost::role_id: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAT86C/InXrgDtXCc9NFze91YMvjTNDqWgv4uzPFI48clOeQyD6x+vOHWP2yNp1OyNHcYLCiLyrv+rSIQyXlLnbeyZWV+7kXIon057Tp7l0BxWtd0hjQEcyWzqQQE7R264C8/qKRak81LIu6RshWZAchYo/BMPuOqVr0m+1zDwOV9JwZc3bpexzsl57CK5pesOrpfdvnd/xrOoEMR+P+C5PC6QLtQl3zkOD3N9kP6HqwbhWH5ZBPy88Kc+5kYM6QVpQSjFIIHK1SWsN0VZoxpkuFlFXB5KHDgZtg3kxrofzjQghl41zJBCDq9Z5oZ+2b1p/j/9jCASyp/ju68H5WXzbzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCf4Nqp6SAl/XjmhPDnTvVJgDCdDhxWaChhjJ3eRcW4NTFgf3zm7Bu65za0li26FKuKks00duF4zebfNw7ZUVsYtIU=]

hieradata/roles/infra/puppet/master.yaml

@@ -37,6 +37,14 @@ profiles::helpers::certmanager::vault_config:
   output_path: '/tmp/certmanager'
   role_id: "%{lookup('certmanager::role_id')}"
 
+profiles::helpers::sshsignhost::vault_config:
+  addr: 'https://vault.service.consul:8200'
+  mount_point: 'ssh-host-signer'
+  approle_path: 'approle'
+  role_name: 'hostrole'
+  output_path: '/tmp/sshsignhost'
+  role_id: "%{lookup('sshsignhost::role_id')}"
+
 profiles::puppet::server::agent_server: 'puppet.query.consul'
 profiles::puppet::server::report_server: 'puppet.query.consul'
 profiles::puppet::server::ca_server: 'puppetca.query.consul'

modules/libs/lib/facter/sshd_host_cert_exists.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# lib/facter/sshd_host_cert_exists.rb
+require 'puppet'
+
+Facter.add('sshd_host_cert_exists') do
+  setcode do
+    File.exist?('/etc/ssh/ssh_host_rsa_key-cert.pem')
+  end
+end

modules/libs/lib/facter/sshd_host_principals.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# lib/facter/sshd_host_principals.rb
+require 'puppet'
+
+Facter.add('sshd_host_principals') do
+  setcode do
+    principals_file = '/etc/ssh/host_principals'
+    if File.exist?(principals_file)
+      File.read(principals_file).split("\n")
+    else
+      []
+    end
+  end
+end

site/profiles/manifests/base.pp

@@ -32,6 +32,8 @@ class profiles::base (
     include profiles::ntp::client
     include profiles::dns::base
     include profiles::pki::vault
+    include profiles::ssh::sign
+    include profiles::ssh::knownhosts
     include profiles::cloudinit::init
     include profiles::metrics::default
     include profiles::helpers::node_lookup

site/profiles/manifests/helpers/sshsignhost.pp

@@ -0,0 +1,77 @@
+# profiles::helpers::sshsignhost
+#
+# wrapper class for python, pip and venv
+class profiles::helpers::sshsignhost (
+  String                $script_name = 'sshsignhost',
+  Stdlib::AbsolutePath  $base_path = "/opt/${script_name}",
+  Stdlib::AbsolutePath  $venv_path = "${base_path}/venv",
+  Stdlib::AbsolutePath  $config_path = "${base_path}/config.yaml",
+  Hash                  $vault_config = {},
+  String                $owner = 'root',
+  String                $group = 'root',
+  Boolean               $systempkgs = false,
+  String                $version = 'system',
+  Array[String[1]]      $packages = ['requests', 'pyyaml'],
+){
+
+  if $::facts['python3_version'] {
+
+    $python_version = $version ? {
+      'system' => $::facts['python3_version'],
+      default  => $version,
+    }
+
+    # ensure the base_path exists
+    file { $base_path:
+      ensure => directory,
+      mode   => '0755',
+      owner  => $owner,
+      group  => $group,
+    }
+
+    # create a venv
+    python::pyvenv { $venv_path :
+      ensure     => present,
+      version    => $python_version,
+      systempkgs => $systempkgs,
+      venv_dir   => $venv_path,
+      owner      => $owner,
+      group      => $group,
+      require    => File[$base_path],
+    }
+
+    # install the required pip packages
+    $packages.each |String $package| {
+      python::pip { "${venv_path}_${package}":
+        ensure     => present,
+        pkgname    => $package,
+        virtualenv => $venv_path,
+      }
+    }
+
+    # create the script from a template
+    file { "${base_path}/${script_name}":
+      ensure  => file,
+      mode    => '0755',
+      content => template("profiles/helpers/${script_name}.erb"),
+      require => Python::Pyvenv[$venv_path],
+    }
+
+    # create the config from a template
+    file { $config_path:
+      ensure  => file,
+      mode    => '0660',
+      owner   => 'puppet',
+      group   => 'root',
+      content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
+      require => Python::Pyvenv[$venv_path],
+    }
+
+    # create symbolic link in $PATH
+    file { "/usr/local/bin/${script_name}":
+      ensure  => 'link',
+      target  => "${base_path}/${script_name}",
+      require => File["${base_path}/${script_name}"],
+    }
+  }
+}

site/profiles/manifests/puppet/puppetmaster.pp

@@ -15,6 +15,7 @@ class profiles::puppet::puppetmaster (
     include profiles::puppet::autosign
     include profiles::puppet::gems
     include profiles::helpers::certmanager
+    include profiles::helpers::sshsignhost
     include profiles::puppet::server
     include profiles::puppet::puppetca
     include profiles::puppet::eyaml

site/profiles/manifests/ssh/knownhosts.pp

@@ -0,0 +1,12 @@
+# manage known hosts
+class profiles::ssh::knownhosts (
+  Array $lines = [],
+) {
+  file {'/etc/ssh/ssh_known_hosts':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('profiles/ssh/ssh_known_hosts.erb'),
+  }
+}

site/profiles/manifests/ssh/sign.pp

@@ -0,0 +1,84 @@
+# profiles::ssh::sign
+class profiles::ssh::sign (
+  Optional[Array[Stdlib::Host]] $principals = [],
+){
+
+  # validate and prepare additional alt_names, if any
+  $default_principals = [
+    $::facts['networking']['hostname'],
+    $::facts['networking']['fqdn'],
+    $::facts['networking']['ip'],
+  ]
+  $effective_principals = $principals ? {
+    []      => $default_principals,
+    default => concat($default_principals, $principals),
+  }
+
+  # path for the principals file
+  $principals_file = '/etc/ssh/host_principals'
+
+  # alt_names_file contents
+  $principals_file_content = $effective_principals
+
+  # manage the alt names file
+  file { $principals_file:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => join($principals_file_content, "\n"),
+  }
+
+  # compare the sorted arrays of principals from disk (fact) vs what is intended (this run)
+  $principals_match = sort($::facts['sshd_host_principals']) == sort($principals_file_content)
+
+  # only renew signed certificate if doesnt exist or the principals have changed
+  if ! $::facts['sshd_host_cert_exists'] or ! $principals_match {
+
+    $common_name = $::facts['networking']['fqdn']
+    $valid_hours = '87600h'
+
+    # prepare alt_names and ip_sans arguments conditionally
+    $principals_string = $effective_principals.empty() ? {
+      true    => '',
+      default => join($effective_principals, ','),
+    }
+
+    # sshsignhost arguments
+    $cmd = '/usr/local/bin/sshsignhost'
+    $principals_arg = '--valid_principals'
+    $ttl_arg = '--ttl'
+    $public_key_arg = '--public_key'
+
+    # call the script with generate(), capturing json output
+    $json_output = generate(
+      $cmd,
+      $principals_arg,
+      $principals_string,
+      $ttl_arg,
+      $valid_hours,
+      $public_key_arg,
+      "${facts['ssh']['rsa']['type']} ${facts['ssh']['rsa']['key']}",
+      '--json'
+    )
+    $signed_data = parsejson($json_output)
+
+    # manage the signed hostkey file
+    file { '/etc/ssh/ssh_host_rsa_key-cert.pem':
+      ensure  => file,
+      content => $signed_data['signed_key'],
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+    }
+
+  }else{
+    # manage the signed hostkey file
+    file { '/etc/ssh/ssh_host_rsa_key-cert.pem':
+      ensure => file,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0644',
+    }
+  }
+}

site/profiles/templates/helpers/sshsignhost.erb

@@ -0,0 +1,83 @@
+#!<%= @venv_path %>/bin/python
+import argparse
+import requests
+import json
+import yaml
+
+# remove this after certs are generated everywhere
+requests.packages.urllib3.disable_warnings()
+
+def load_config(config_path):
+    with open(config_path, 'r') as file:
+        config = yaml.safe_load(file)
+    return config['vault']
+
+def authenticate_approle(vault_config):
+    url = f"{vault_config['addr']}/v1/auth/{vault_config['approle_path']}/login"
+    payload = {
+        "role_id": vault_config['role_id'],
+    }
+    response = requests.post(url, json=payload, verify=False)
+    if response.status_code == 200:
+        auth_response = response.json()
+        return auth_response['auth']['client_token']
+    else:
+        print(f"Error authenticating with AppRole: {response.text}")
+        return None
+
+def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
+    # Authenticate using AppRole and get a token
+    client_token = authenticate_approle(vault_config)
+    if not client_token:
+        print("Failed to authenticate with Vault using AppRole.")
+        return None
+
+    # Prepare the SSH certificate signing request
+    url = f"{vault_config['addr']}/v1/{vault_config['mount_point']}/sign/{vault_config['role_name']}"
+    headers = {'X-Vault-Token': client_token}
+    payload = {
+        "cert_type": "host",
+        "public_key": public_key,
+        "valid_principals": valid_principals,
+        "ttl": ttl
+    }
+
+    # Request the SSH certificate signing
+    response = requests.post(url, headers=headers, json=payload, verify=False)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        print(f"Error requesting certificate: {response.text}")
+        return None
+
+def main(config_file):
+    config = load_config(config_file)
+    parser = argparse.ArgumentParser(description='Sign SSH host certificate using Vault.')
+    parser.add_argument('--public_key', required=True, help='SSH public key as a string')
+    parser.add_argument('--valid_principals', required=True, help='Comma-separated list of valid principals')
+    parser.add_argument('--ttl', default='87600h', help='Time-to-live for the certificate (default: 87600h)')
+    parser.add_argument('--json', action='store_true', help='Output the resulting certificate as JSON')
+
+    args = parser.parse_args()
+
+    # Load configuration
+    config = load_config(config_file)
+
+    # Sign SSH certificate
+    response = sign_ssh_certificate(config, args.public_key, args.valid_principals, args.ttl)
+
+    if response and 'data' in response and 'signed_key' in response['data']:
+        if args.json:
+            output = {
+                'signed_key': response['data']['signed_key'],
+            }
+            print(json.dumps(output))
+        else:
+            print(response['data']['signed_key'])
+    else:
+        print("Error: The response does not contain the expected data.")
+        exit(1)
+
+if __name__ == "__main__":
+    config_file = '<%= @config_path %>'
+    main(config_file)

site/profiles/templates/helpers/sshsignhost_config.yaml.erb

@@ -0,0 +1,7 @@
+vault:
+  addr: '<%= @vault_config['addr'] %>'
+  role_id: '<%= @vault_config['role_id'] %>'
+  approle_path: '<%= @vault_config['approle_path'] %>'
+  mount_point: '<%= @vault_config['mount_point'] %>'
+  role_name: '<%= @vault_config['role_name'] %>'
+output_path: '<%= @vault_config['output_path'] %>'

site/profiles/templates/ssh/ssh_known_hosts.erb

@@ -0,0 +1,4 @@
+# this file is managed by puppet
+<% @lines.each do |line| -%>
+<%= line %>
+<% end -%>