Run dnsmasq as the mail edge local resolver

WAN federation through mesh gateways makes Consul verify server certs against `server.<dc>.consul` and `<node_name>.server.<dc>.consul`. Node names are FQDNs, so each server needs its exact SAN. This adds the SANs ahead of Consul server TLS, which a stacked follow-up enables once the reissued certs are verified.

- add `server.<country>-<region>.consul` to consul server alt_names
- add `<fqdn>.server.<country>-<region>.consul` to consul server alt_names

Reviewed-on: #547
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

doc/vault/README.md

@@ -88,36 +88,13 @@
 
 # SSH Hostkey Signing
 
-## create ssh engine, key, set ttl
-    vault secrets enable -path=ssh-host-signer ssh
-    vault write ssh-host-signer/config/ca generate_signing_key=true
-    vault secrets tune -max-lease-ttl=87600h ssh-host-signer
+The `sshca` ssh engine, its `signhost` role, the `sshca/sign/signhost` policy and
+the `sshsigner` approle are managed in terraform-vault:
 
-## create role
-    vault write ssh-host-signer/roles/hostrole \
-        key_type=ca \
-        algorithm_signer=rsa-sha2-256 \
-        ttl=87600h \
-        allow_host_certificates=true \
-        allowed_domains="unkin.net" \
-        allow_subdomains=true \
-        allow_baredomains=true
+- `config/ssh_secret_backend/sshca.yaml`
+- `config/ssh_secret_backend_role/sshca/signhost.yaml`
+- `config/auth_approle_role/approle/sshsigner.yaml`
+- `policies/sshca/sign/signhost.yaml`
 
-## create policy to use hostrole
-    cat <<EOF > sshsign-host.hcl
-    path "ssh-host-signer/sign/hostrole" {
-        capabilities = ["create", "update"]
-    }
-    EOF
-
-    vault policy write sshsign-host-policy sshsign-host.hcl
-
-    vault write auth/approle/role/sshsign-host-role \
-        bind_secret_id=false \
-        token_policies="sshsign-host-policy" \
-        token_ttl=30s \
-        token_max_ttl=30s \
-        token_bound_cidrs="198.18.17.3/32,198.18.13.32/32,198.18.13.33/32,198.18.13.34/32"
-
-## get the sshsign-host-role approle id
-    vault read -field=role_id auth/approle/role/sshsign-host-role/role-id
+## get the sshsigner approle id
+    vault read -field=role_id auth/approle/role/sshsigner/role-id

hieradata/common.yaml

@@ -178,6 +178,8 @@ lookup_options:
     convert_to: Sensitive
   stalwart::fallback_admin_password:
     convert_to: Sensitive
+  wireguard::interfaces:
+    convert_to: Sensitive
 
 facts_path: '/opt/puppetlabs/facter/facts.d'
 
@@ -401,7 +403,7 @@ networking::route_defaults:
 
 # logging:
 victorialogs::client::journald::enable: true
-victorialogs::client::journald::inserturl: https://logs-ingest.k8s.syd1.au.unkin.net/insert/journald
+victorialogs::client::journald::inserturl: https://logs-ingest.k8s.syd1.au.unkin.net:443/insert/journald
 
 # FIXME these are for the proxmox ceph cluster
 profiles::ceph::client::fsid: 7f7f00cb-95de-498c-8dcc-14b54e4e9ca8

hieradata/country/au/region/syd1.yaml

@@ -7,3 +7,11 @@ profiles_dns_upstream_forwarder_consul:
   - 198.18.19.14
 profiles_dns_upstream_forwarder_k8s:
   - 198.18.19.20
+profiles::consul::client::members_lookup: false
+# static: k8s-compiled hosts can't see the servers in PuppetDB; update when servers change
+profiles::consul::client::consul_servers:
+  - ausyd1nxvm2005.main.unkin.net
+  - ausyd1nxvm2006.main.unkin.net
+  - ausyd1nxvm2007.main.unkin.net
+  - ausyd1nxvm2008.main.unkin.net
+  - ausyd1nxvm2009.main.unkin.net

hieradata/country/au/region/syd1/infra/halb/haproxy2.yaml

@@ -3,23 +3,7 @@ haproxy_server_k8s_syd1_traefik_internal: 'k8s-traefik-internal 198.18.200.4:443
 haproxy_server_k8s_syd1_traefik_external: 'k8s-traefik-external 198.18.199.0:443 ssl verify none check inter 2s rise 3 fall 2'
 
 profiles::haproxy::dns::ipaddr: "%{hiera('anycast_ip')}"
-profiles::haproxy::dns::vrrp_cnames:
-  - sonarr.main.unkin.net
-  - radarr.main.unkin.net
-  - lidarr.main.unkin.net
-  - readarr.main.unkin.net
-  - prowlarr.main.unkin.net
-  - nzbget.main.unkin.net
-  - git.unkin.net
-  - fafflix.unkin.net
-  - grafana.unkin.net
-  - dashboard.ceph.unkin.net
-  - mail-webadmin.main.unkin.net
-  - mail-in.main.unkin.net
-  - mail.main.unkin.net
-  - autoconfig.main.unkin.net
-  - autodiscover.main.unkin.net
-  - auth.unkin.net
+profiles::haproxy::dns::vrrp_cnames: []
 
 profiles::haproxy::mappings:
   fe_http:
@@ -432,9 +416,7 @@ profiles::pki::vault::alt_names:
   - mail-webadmin.main.unkin.net
 
 # additional cnames
-profiles::haproxy::dns::cnames:
-  - au-syd1-pve.main.unkin.net
-  - au-syd1-pve-api.main.unkin.net
+profiles::haproxy::dns::cnames: []
 
 # letsencrypt certificates
 certbot::client::service: haproxy

hieradata/nodes/aumel1nxvm2132.main.unkin.net.yaml

@@ -0,0 +1,2 @@
+---
+profiles::postfix::gateway::myhostname: 'mta2.edge.unkin.net'

hieradata/nodes/ausyd2nxvm2130.main.unkin.net.yaml

@@ -0,0 +1,2 @@
+---
+profiles::postfix::gateway::myhostname: 'mta1.edge.unkin.net'

hieradata/nodes/prodnxsr0020.main.unkin.net.yaml

@@ -0,0 +1,45 @@
+---
+# primary interface is the WAN uplink; pin host identity to the dum0 loopback
+networking_loopback0_ip: 198.18.2.160
+networking_loopback1_ip: 198.18.21.160
+
+# dns: keep the local dnsmasq resolver
+profiles::dns::base::nameservers:
+  - 127.0.0.1
+profiles::dns::base::search:
+  - main.unkin.net
+profiles::dns::base::primary_interface: dum0
+profiles::dns::updater::deny_ranges:
+  - 198.18.199.0/24
+  - 198.18.200.0/24
+  - 10.42.0.0/16
+  - 10.43.0.0/16
+  - 10.10.12.0/24     # wg0
+  - 103.216.190.0/23  # wan uplink
+profiles::consul::client::host_addr: "%{hiera('networking_loopback0_ip')}"
+
+# ssh: listen on localhost and dum0 only; knock out the common wan primary ip
+lookup_options:
+  ssh::server::options:
+    merge:
+      strategy: deep
+      knockout_prefix: '--'
+ssh::server::options:
+  ListenAddress:
+    - "--%{facts.networking.ip}"
+    - 127.0.0.1
+    - "%{hiera('networking_loopback0_ip')}"
+profiles::ssh::sign::principals:
+  - "%{hiera('networking_loopback0_ip')}"
+
+# frrouting
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_interfaces:
+  dum0:
+    area: 0.0.0.0
+  dum1:
+    area: 0.0.0.0
+  bond0.201:
+    area: 0.0.0.0
+frrouting::ospf_preferred_source_enable: true
+frrouting::ospf_preferred_source: "%{hiera('networking_loopback1_ip')}"

hieradata/os/AlmaLinux/all_releases.yaml

@@ -147,6 +147,16 @@ profiles::yum::global::repos:
     baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/postgresql/common/redhat/rhel-%{facts.os.release.major}-%{facts.os.architecture}
     gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/postgresql/keys/PGDG-RPM-GPG-KEY-RHEL
 
+  # Rspamd repositories
+  rspamd:
+    ensure: absent
+    name: rspamd
+    descr: rspamd repository
+    target: /etc/yum.repos.d/rspamd.repo
+    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rspamd/rpm-stable/centos-%{facts.os.release.major}/%{facts.os.architecture}
+    gpgkey: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/rspamd/rpm/rspamd.asc
+    mirrorlist: absent
+
   # Ceph repositories
   ceph:
     ensure: absent

hieradata/roles/apps/media.yaml

@@ -2,6 +2,8 @@
 hiera_include:
   - profiles::nginx::simpleproxy
 
+profiles::puppet::migrate::enabled: true
+
 profiles::yum::global::repos:
   ceph:
     ensure: present

hieradata/roles/infra/incus/node.yaml

@@ -14,6 +14,8 @@ hiera_include:
   - profiles::storage::cephfsvols
   - exporters::frr_exporter
 
+profiles::puppet::migrate::enabled: true
+
 # FIXME: puppet-python wants to try manage python-dev, which is required by the ceph package
 python::manage_dev_package: false
 

hieradata/roles/infra/k8s.yaml

@@ -8,6 +8,8 @@ hiera_include:
   - frrouting
   - rke2
 
+profiles::puppet::migrate::enabled: true
+
 # manage rke2
 rke2::bootstrap_node: prodnxsr0001.main.unkin.net
 rke2::join_url: https://join-k8s.service.consul:9345

hieradata/roles/infra/mail/gateway.yaml

@@ -1,26 +1,81 @@
 ---
+hiera_include:
+  - redis
 
-# additional altnames
-profiles::pki::vault::alt_names:
-  - in-mta.main.unkin.net
+# local caching resolver for postfix and rspamd, forwarding to the lab resolver over the vpn route
+dnsmasq::no_resolv: true
+dnsmasq::cache_size: 10000
+dnsmasq::servers:
+  - 198.18.200.7
+profiles::dns::base::nameservers:
+  - 127.0.0.1
+
+profiles::yum::global::repos:
+  rspamd:
+    ensure: present
+
+# local redis for rspamd bayes and history; bayes tokens carry a ttl so volatile-ttl can evict
+redis::maxmemory: '128mb'
+redis::maxmemory_policy: 'volatile-ttl'
+
+# local rspamd, deterministic checks only; module defaults cover spf/dkim/dmarc/arc/rbl/phishing/mime_types/fuzzy_check
+profiles::rspamd::edge::rules:
+  edge.cf: 'https://git.unkin.net/unkin/rspamd-rules/raw/branch/main/edge.cf'
+profiles::rspamd::edge::local_d:
+  worker-proxy.inc: |
+    bind_socket = "127.0.0.1:11332";
+    milter = yes;
+    timeout = 120s;
+    count = 1;
+    upstream "local" {
+      default = yes;
+      self_scan = yes;
+    }
+  worker-normal.inc: |
+    enabled = false;
+  redis.conf: |
+    servers = "127.0.0.1:6379";
+  actions.conf: |
+    reject = 13;
+    add_header = 6;
+    rewrite_subject = null;
+    greylist = null;
+  greylist.conf: |
+    enabled = false;
+  gpt.conf: |
+    enabled = false;
+  classifier-bayes.conf: |
+    backend = "redis";
+    expire = 8640000;
+    autolearn {
+      spam_threshold = 6.0;
+      ham_threshold = -0.5;
+      check_balance = true;
+    }
+  spamassassin.conf: |
+    ruleset = "/etc/rspamd/rules/*.cf";
+    alpha = 0.1;
+  settings.conf: |
+    smarthost_outbound {
+      priority = high;
+      ip = "103.216.191.185";
+      want_spam = yes;
+    }
 
 # base postfix configuration (passed to postfix class)
 postfix::relayhost: 'direct'
-postfix::myorigin: 'main.unkin.net'
+postfix::myorigin: 'unkin.net'
 postfix::mydestination: 'blank'
-postfix::mynetworks: '127.0.0.0/8 [::1]/128'
+postfix::mynetworks: '127.0.0.0/8 [::1]/128 103.216.191.185/32'
 postfix::mta: true
 postfix::manage_aliases: true
 
-# profile parameters for customization
-profiles::postfix::gateway::myhostname: 'in-mta.main.unkin.net'
-
 # postfix map content (templates)
 profiles::postfix::gateway::relay_recipients_maps:
-  '@main.unkin.net': 'OK'
+  '@unkin.net': 'OK'
 
 profiles::postfix::gateway::relay_domains_maps:
-  'main.unkin.net': 'OK'
+  'unkin.net': 'OK'
 
 profiles::postfix::gateway::postscreen_access_maps:
   '127.0.0.1/32': 'permit'
@@ -32,21 +87,24 @@ profiles::postfix::gateway::helo_access_maps:
   'unknown': 'REJECT'
   'localhost': 'REJECT You are not localhost'
 
+profiles::postfix::gateway::smtp_tls_policy_maps:
+  '[mail-in.main.unkin.net]:25': 'secure match=hostname'
+
 # postfix transports
 postfix::transports:
-  'main.unkin.net':
+  'unkin.net':
     ensure: present
     destination: 'relay'
-    nexthop: 'mail-in.main.unkin.net:25'
+    nexthop: '[mail-in.main.unkin.net]:25'
 
 # postfix virtuals
 postfix::virtuals:
   'root':
     ensure: present
-    destination: 'ben@main.unkin.net'
+    destination: 'ben@unkin.net'
   'postmaster':
     ensure: present
-    destination: 'ben@main.unkin.net'
+    destination: 'ben@unkin.net'
   'abuse':
     ensure: present
-    destination: 'ben@main.unkin.net'
+    destination: 'ben@unkin.net'

hieradata/roles/infra/network/router.yaml

@@ -0,0 +1,30 @@
+---
+hiera_include:
+  - frrouting
+  - exporters::frr_exporter
+
+# routing
+sysctl::base::values:
+  net.ipv4.ip_forward:
+    value: '1'
+  net.ipv4.conf.all.rp_filter:
+    value: '0'
+  net.ipv4.conf.default.rp_filter:
+    value: '0'
+  # overrides 50-redhat.conf's per-interface rp_filter=1 (applied by udev on link add); sysctl -n can't glob, so no enforce
+  net.ipv4.conf.*.rp_filter:
+    value: '0'
+    enforce: false
+
+# frrouting
+exporters::frr_exporter::enable: true
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::daemons:
+  ospfd: true
+
+# consul
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: frr_exporter
+    disposition: write

hieradata/roles/infra/puppet/master.yaml

@@ -1,5 +1,6 @@
 ---
 profiles::puppet::autosign::subnet_ranges:
+  - '198.18.2.0/24'
   - '198.18.13.0/24'
   - '198.18.14.0/24'
   - '198.18.15.0/24'
@@ -31,6 +32,10 @@ profiles::puppet::enc::repo: https://git.service.au-syd1.consul/unkinben/puppet-
 profiles::packages::include:
   encapic:
     ensure: '0.2.0'
+  certmanager:
+    ensure: '0.2.0'
+  sshsignhost:
+    ensure: '0.1.0'
 
 profiles::puppet::encapic::encapi_url: https://encapi.k8s.syd1.au.unkin.net
 profiles::puppet::server::external_nodes: '/usr/bin/encapic-enc'
@@ -56,10 +61,9 @@ profiles::helpers::certmanager::vault_config:
 
 profiles::helpers::sshsignhost::vault_config:
   addr: 'https://vault.service.consul:8200'
-  mount_point: 'ssh-host-signer'
+  mount_point: 'sshca'
   approle_path: 'approle'
-  role_name: 'hostrole'
-  output_path: '/tmp/sshsignhost'
+  role_name: 'signhost'
   role_id: "%{lookup('sshsignhost::role_id')}"
 
 profiles::puppet::server::agent_server: 'puppet.query.consul'

hieradata/roles/infra/storage/consul.yaml

@@ -31,6 +31,8 @@ profiles::pki::vault::alt_names:
   - consul.service.consul
   - "consul.service.%{facts.country}-%{facts.region}.consul"
   - consul
+  - "server.%{facts.country}-%{facts.region}.consul"
+  - "%{facts.networking.fqdn}.server.%{facts.country}-%{facts.region}.consul"
 
 # manage a simple nginx reverse proxy
 profiles::nginx::simpleproxy::nginx_vhost: 'consul.service.consul'

modules/dnsmasq/manifests/init.pp

@@ -0,0 +1,51 @@
+# manage dnsmasq as a dns forwarder and dhcp relay
+class dnsmasq (
+  Boolean $manage_package                    = true,
+  Boolean $manage_service                    = true,
+  String  $package_name                      = 'dnsmasq',
+  String  $service_name                      = 'dnsmasq',
+  Stdlib::Absolutepath $config_file          = '/etc/dnsmasq.conf',
+  Stdlib::Absolutepath $config_dir           = '/etc/dnsmasq.d',
+  Boolean $purge_config_dir                  = false,
+  Array[String] $interfaces                  = [],
+  Array[Stdlib::IP::Address] $listen_addresses = ['127.0.0.1'],
+  Enum['bind-interfaces', 'bind-dynamic', 'none'] $bind_mode = 'bind-interfaces',
+  Boolean $no_resolv                         = false,
+  Array[String] $servers                     = [],
+  Hash[String, Array[String]] $forwards      = {},
+  Optional[Integer[0]] $cache_size           = undef,
+  Boolean $domain_needed                     = true,
+  Boolean $bogus_priv                        = true,
+  Array[String] $dhcp_relays                 = [],
+  Array[String] $options                     = [],
+) {
+
+  if $manage_package {
+    package { $package_name:
+      ensure => installed,
+      before => File[$config_file, $config_dir],
+    }
+  }
+
+  file { $config_dir:
+    ensure  => directory,
+    recurse => $purge_config_dir,
+    purge   => $purge_config_dir,
+  }
+
+  file { $config_file:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('dnsmasq/dnsmasq.conf.erb'),
+  }
+
+  if $manage_service {
+    service { $service_name:
+      ensure    => running,
+      enable    => true,
+      subscribe => File[$config_file, $config_dir],
+    }
+  }
+}

modules/dnsmasq/templates/dnsmasq.conf.erb

@@ -0,0 +1,40 @@
+# THIS FILE IS MANAGED BY PUPPET
+user=dnsmasq
+group=dnsmasq
+conf-dir=<%= @config_dir %>,.rpmnew,.rpmsave,.rpmorig
+
+<% @interfaces.each do |iface| -%>
+interface=<%= iface %>
+<% end -%>
+<% unless @listen_addresses.empty? -%>
+listen-address=<%= @listen_addresses.join(',') %>
+<% end -%>
+<% unless @bind_mode == 'none' -%>
+<%= @bind_mode %>
+<% end -%>
+<% if @no_resolv -%>
+no-resolv
+<% end -%>
+<% if @domain_needed -%>
+domain-needed
+<% end -%>
+<% if @bogus_priv -%>
+bogus-priv
+<% end -%>
+<% if @cache_size -%>
+cache-size=<%= @cache_size %>
+<% end -%>
+<% @servers.each do |server| -%>
+server=<%= server %>
+<% end -%>
+<% @forwards.keys.sort.each do |domain| -%>
+<% @forwards[domain].each do |server| -%>
+server=/<%= domain %>/<%= server %>
+<% end -%>
+<% end -%>
+<% @dhcp_relays.each do |relay| -%>
+dhcp-relay=<%= relay %>
+<% end -%>
+<% @options.each do |line| -%>
+<%= line %>
+<% end -%>

modules/libs/lib/facter/subnet_facts.rb

@@ -5,6 +5,7 @@ require 'ipaddr'
 # a class that creates facts based on the subnet
 class SubnetAttributes
   SUBNET_TO_ATTRIBUTES = {
+    '198.18.2.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' }, # router loopbacks
     '198.18.13.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' },
     '198.18.14.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' },
     '198.18.15.0/24' => { environment: 'prod', region: 'syd1', country: 'au', zone: 'common' },

modules/nzbget/manifests/init.pp

@@ -6,8 +6,8 @@ class nzbget (
   $manage_group       = $nzbget::params::manage_group,
   $service_enable     = $nzbget::params::service_enable,
   $service_name       = $nzbget::params::service_name,
-  $bind_address       = $sonarr::params::bind_address,
-  $port               = $sonarr::params::port,
+  $bind_address       = $nzbget::params::bind_address,
+  $port               = $nzbget::params::port,
 ) inherits nzbget::params {
 
   include nzbget::install

modules/wireguard/manifests/init.pp

@@ -0,0 +1,44 @@
+# manage wireguard interfaces via wg-quick
+class wireguard (
+  Boolean                        $manage_package = true,
+  String                         $package_name   = 'wireguard-tools',
+  Variant[Hash, Sensitive[Hash]] $interfaces = {},
+) {
+
+  if $manage_package {
+    package { $package_name:
+      ensure => installed,
+      before => File['/etc/wireguard'],
+    }
+  }
+
+  file { '/etc/wireguard':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0700',
+  }
+
+  # hiera hands eyaml secrets over as plain strings inside the (Sensitive) hash; re-wrap them per resource
+  $raw = $interfaces ? {
+    Sensitive => $interfaces.unwrap,
+    default   => $interfaces,
+  }
+
+  $raw.each |String $iface, Hash $data| {
+    $peers = $data.get('peers', []).map |Hash $peer| {
+      $peer['preshared_key'] =~ String ? {
+        true    => $peer + { 'preshared_key' => Sensitive($peer['preshared_key']) },
+        default => $peer,
+      }
+    }
+    $private_key = $data['private_key'] =~ String ? {
+      true    => Sensitive($data['private_key']),
+      default => $data['private_key'],
+    }
+
+    wireguard::interface { $iface:
+      * => $data + { 'peers' => $peers, 'private_key' => $private_key },
+    }
+  }
+}

modules/wireguard/manifests/interface.pp

@@ -0,0 +1,63 @@
+# manage one wg-quick interface; without private_key, /etc/wireguard/<iface>.key is generated once and loaded via PostUp
+define wireguard::interface (
+  Array[Stdlib::IP::Address]           $addresses,
+  Optional[Stdlib::Port]               $listen_port = undef,
+  Optional[Integer[1280, 9000]]        $mtu         = undef,
+  Optional[Sensitive[String[1]]]       $private_key = undef,
+  Array[Struct[{
+    public_key           => String[1],
+    allowed_ips          => Variant[String[1], Array[String[1], 1]],
+    preshared_key        => Optional[Sensitive[String[1]]],
+    endpoint             => Optional[String[1]],
+    persistent_keepalive => Optional[Integer[0, 65535]],
+  }]]                                  $peers       = [],
+) {
+
+  $conf = "/etc/wireguard/${name}.conf"
+  $key  = $private_key.then |$k| { $k.unwrap }
+
+  if $private_key =~ Undef {
+    $keyfile = "/etc/wireguard/${name}.key"
+
+    exec { "wireguard_genkey_${name}":
+      command => "/bin/sh -c 'umask 077; wg genkey > ${keyfile}'",
+      creates => $keyfile,
+      path    => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'],
+      require => File['/etc/wireguard'],
+    }
+
+    file { $keyfile:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0600',
+      require => Exec["wireguard_genkey_${name}"],
+      before  => [File[$conf], Service["wg-quick@${name}"]],
+    }
+  }
+
+  file { $conf:
+    ensure    => file,
+    owner     => 'root',
+    group     => 'root',
+    mode      => '0600',
+    content   => Sensitive(template('wireguard/wg.conf.erb')),
+    show_diff => false,
+    notify    => Exec["wireguard_syncconf_${name}"],
+  }
+
+  service { "wg-quick@${name}":
+    ensure  => running,
+    enable  => true,
+    require => File[$conf],
+  }
+
+  # syncconf applies peer/key changes without bouncing the tunnel; address/mtu changes need a manual restart
+  exec { "wireguard_syncconf_${name}":
+    command     => "/bin/bash -c 'wg syncconf ${name} <(wg-quick strip ${name})'",
+    onlyif      => "/usr/sbin/ip link show ${name}",
+    path        => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'],
+    refreshonly => true,
+    require     => Service["wg-quick@${name}"],
+  }
+}

modules/wireguard/templates/wg.conf.erb

@@ -0,0 +1,31 @@
+# THIS FILE IS MANAGED BY PUPPET
+[Interface]
+<% @addresses.each do |addr| -%>
+Address = <%= addr %>
+<% end -%>
+<% if @listen_port -%>
+ListenPort = <%= @listen_port %>
+<% end -%>
+<% if @mtu -%>
+MTU = <%= @mtu %>
+<% end -%>
+<% if @key -%>
+PrivateKey = <%= @key %>
+<% else -%>
+PostUp = wg set %i private-key /etc/wireguard/%i.key
+<% end -%>
+<% @peers.each do |peer| -%>
+
+[Peer]
+PublicKey = <%= peer['public_key'] %>
+<% if peer['preshared_key'] -%>
+PresharedKey = <%= peer['preshared_key'].unwrap %>
+<% end -%>
+AllowedIPs = <%= Array(peer['allowed_ips']).join(', ') %>
+<% if peer['endpoint'] -%>
+Endpoint = <%= peer['endpoint'] %>
+<% end -%>
+<% if peer['persistent_keepalive'] -%>
+PersistentKeepalive = <%= peer['persistent_keepalive'] %>
+<% end -%>
+<% end -%>

site/profiles/manifests/helpers/certmanager.pp

@@ -1,77 +1,28 @@
 # profiles::helpers::certmanager
 #
-# wrapper class for python, pip and venv
+# renders the config.yaml read by the certmanager binary (RPM-installed)
 class profiles::helpers::certmanager (
   String                $script_name = 'certmanager',
   Stdlib::AbsolutePath  $base_path = "/opt/${script_name}",
-  Stdlib::AbsolutePath  $venv_path = "${base_path}/venv",
   Stdlib::AbsolutePath  $config_path = "${base_path}/config.yaml",
   Hash                  $vault_config = {},
   String                $owner = 'root',
   String                $group = 'root',
-  Boolean               $systempkgs = false,
-  String                $version = 'system',
-  Array[String[1]]      $packages = ['requests', 'pyyaml'],
 ){
 
-  if $::facts['python3_version'] {
+  file { $base_path:
+    ensure => directory,
+    mode   => '0755',
+    owner  => $owner,
+    group  => $group,
+  }
 
-    $python_version = $version ? {
-      'system' => $::facts['python3_version'],
-      default  => $version,
-    }
-
-    # ensure the base_path exists
-    file { $base_path:
-      ensure => directory,
-      mode   => '0755',
-      owner  => $owner,
-      group  => $group,
-    }
-
-    # create a venv
-    python::pyvenv { $venv_path :
-      ensure     => present,
-      version    => $python_version,
-      systempkgs => $systempkgs,
-      venv_dir   => $venv_path,
-      owner      => $owner,
-      group      => $group,
-      require    => File[$base_path],
-    }
-
-    # install the required pip packages
-    $packages.each |String $package| {
-      python::pip { "${venv_path}_${package}":
-        ensure     => present,
-        pkgname    => $package,
-        virtualenv => $venv_path,
-      }
-    }
-
-    # create the script from a template
-    file { "${base_path}/${script_name}":
-      ensure  => file,
-      mode    => '0755',
-      content => template("profiles/helpers/${script_name}.erb"),
-      require => Python::Pyvenv[$venv_path],
-    }
-
-    # create the config from a template
-    file { $config_path:
-      ensure  => file,
-      mode    => '0660',
-      owner   => 'puppet',
-      group   => 'root',
-      content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
-      require => Python::Pyvenv[$venv_path],
-    }
-
-    # create symbolic link in $PATH
-    file { "/usr/local/bin/${script_name}":
-      ensure  => 'link',
-      target  => "${base_path}/${script_name}",
-      require => File["${base_path}/${script_name}"],
-    }
+  file { $config_path:
+    ensure  => file,
+    mode    => '0660',
+    owner   => 'puppet',
+    group   => 'root',
+    content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
+    require => File[$base_path],
   }
 }

site/profiles/manifests/helpers/sshsignhost.pp

@@ -1,77 +1,28 @@
 # profiles::helpers::sshsignhost
 #
-# wrapper class for python, pip and venv
+# renders the config.yaml read by the sshsignhost binary (RPM-installed)
 class profiles::helpers::sshsignhost (
   String                $script_name = 'sshsignhost',
   Stdlib::AbsolutePath  $base_path = "/opt/${script_name}",
-  Stdlib::AbsolutePath  $venv_path = "${base_path}/venv",
   Stdlib::AbsolutePath  $config_path = "${base_path}/config.yaml",
   Hash                  $vault_config = {},
   String                $owner = 'root',
   String                $group = 'root',
-  Boolean               $systempkgs = false,
-  String                $version = 'system',
-  Array[String[1]]      $packages = ['requests', 'pyyaml'],
 ){
 
-  if $::facts['python3_version'] {
+  file { $base_path:
+    ensure => directory,
+    mode   => '0755',
+    owner  => $owner,
+    group  => $group,
+  }
 
-    $python_version = $version ? {
-      'system' => $::facts['python3_version'],
-      default  => $version,
-    }
-
-    # ensure the base_path exists
-    file { $base_path:
-      ensure => directory,
-      mode   => '0755',
-      owner  => $owner,
-      group  => $group,
-    }
-
-    # create a venv
-    python::pyvenv { $venv_path :
-      ensure     => present,
-      version    => $python_version,
-      systempkgs => $systempkgs,
-      venv_dir   => $venv_path,
-      owner      => $owner,
-      group      => $group,
-      require    => File[$base_path],
-    }
-
-    # install the required pip packages
-    $packages.each |String $package| {
-      python::pip { "${venv_path}_${package}":
-        ensure     => present,
-        pkgname    => $package,
-        virtualenv => $venv_path,
-      }
-    }
-
-    # create the script from a template
-    file { "${base_path}/${script_name}":
-      ensure  => file,
-      mode    => '0755',
-      content => template("profiles/helpers/${script_name}.erb"),
-      require => Python::Pyvenv[$venv_path],
-    }
-
-    # create the config from a template
-    file { $config_path:
-      ensure  => file,
-      mode    => '0660',
-      owner   => 'puppet',
-      group   => 'root',
-      content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
-      require => Python::Pyvenv[$venv_path],
-    }
-
-    # create symbolic link in $PATH
-    file { "/usr/local/bin/${script_name}":
-      ensure  => 'link',
-      target  => "${base_path}/${script_name}",
-      require => File["${base_path}/${script_name}"],
-    }
+  file { $config_path:
+    ensure  => file,
+    mode    => '0660',
+    owner   => 'puppet',
+    group   => 'root',
+    content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
+    require => File[$base_path],
   }
 }

site/profiles/manifests/postfix/gateway.pp

@@ -3,8 +3,7 @@ class profiles::postfix::gateway (
   Stdlib::Absolutepath $tls_key_file  = '/etc/pki/tls/vault/certificate.pem',
   Stdlib::Absolutepath $tls_ca_file   = '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem',
   String $myhostname                  = $trusted['certname'],
-  String $message_size_limit          = '133169152',
-  String $mailbox_size_limit          = '133169152',
+  String $message_size_limit          = '52428800',
   String $local_transport             = 'error:No local mail delivery',
   Boolean $enable_postscreen          = true,
   Array[String] $alias_maps           = [
@@ -12,12 +11,12 @@ class profiles::postfix::gateway (
     'hash:/etc/postfix/aliases',
   ],
   Array[String] $postscreen_dnsbl_sites = [
-    'zen.spamhaus.org*3',
-    'b.barracudacentral.org=127.0.0.[2..11]*2',
-    'bl.spameatingmonkey.net*2',
-    'bl.spamcop.net',
-    'dnsbl.sorbs.net',
-    'swl.spamhaus.org*-4',
+    'zen.spamhaus.org=127.0.0.[2..11]*3',
+    'b.barracudacentral.org=127.0.0.2*2',
+    'bl.mailspike.net=127.0.0.2*1',
+    'psbl.surriel.com=127.0.0.2*1',
+    'bl.spameatingmonkey.net=127.0.0.2*2',
+    'bl.spamcop.net=127.0.0.2',
     'list.dnswl.org=127.[0..255].[0..255].0*-2',
     'list.dnswl.org=127.[0..255].[0..255].1*-4',
     'list.dnswl.org=127.[0..255].[0..255].[2..3]*-6',
@@ -25,7 +24,6 @@ class profiles::postfix::gateway (
   Array[String] $smtpd_client_restrictions = [
     'permit_sasl_authenticated',
     'permit_mynetworks',
-    'reject_rbl_client zen.spamhaus.org',
   ],
   Array[String] $smtpd_sender_restrictions = [
     'permit_sasl_authenticated',
@@ -47,6 +45,7 @@ class profiles::postfix::gateway (
     'permit_mynetworks',
     'reject_unauth_destination',
   ],
+  String $smtpd_milters = 'inet:127.0.0.1:11332',
   Hash[String, String] $smtp_tls_policy_maps = {},
   Hash[String, String] $sender_canonical_maps = {},
   Hash[String, String] $sender_access_maps = {},
@@ -73,7 +72,7 @@ class profiles::postfix::gateway (
         'value' => 'permit_mynetworks, cidr:/etc/postfix/postscreen_access'
       },
       'postscreen_blacklist_action' => {
-        'value' => 'enforce'
+        'value' => 'drop'
       },
       'postscreen_cache_map' => {
         'value' => 'btree:$data_directory/postscreen_cache'
@@ -87,6 +86,10 @@ class profiles::postfix::gateway (
       'postscreen_dnsbl_threshold' => {
         'value' => '2'
       },
+      # el9 ships postfix 3.5, before the allowlist rename
+      'postscreen_dnsbl_whitelist_threshold' => {
+        'value' => '-1'
+      },
       'postscreen_greet_action' => {
         'value' => 'enforce'
       },
@@ -108,9 +111,6 @@ class profiles::postfix::gateway (
     'alias_database' => {
       'value' => $alias_maps_string
     },
-    'default_destination_recipient_limit' => {
-      'value' => '1'
-    },
     'disable_vrfy_command' => {
       'value' => 'yes'
     },
@@ -129,9 +129,6 @@ class profiles::postfix::gateway (
     'local_transport' => {
       'value' => $local_transport
     },
-    'mailbox_size_limit' => {
-      'value' => $mailbox_size_limit
-    },
     'message_size_limit' => {
       'value' => $message_size_limit
     },
@@ -141,8 +138,20 @@ class profiles::postfix::gateway (
     'non_smtpd_milters' => {
       'ensure' => 'blank'
     },
-    'qmqpd_authorized_clients' => {
-      'value' => '127.0.0.1 [::1]'
+    'milter_connect_timeout' => {
+      'value' => '10s'
+    },
+    'milter_content_timeout' => {
+      'value' => '60s'
+    },
+    'milter_default_action' => {
+      'value' => 'tempfail'
+    },
+    'milter_mail_macros' => {
+      'value' => 'i {mail_addr} {client_addr} {client_name} {auth_authen}'
+    },
+    'milter_protocol' => {
+      'value' => '6'
     },
     'recipient_canonical_maps' => {
       'value' => 'hash:/etc/postfix/recipient_canonical'
@@ -177,7 +186,7 @@ class profiles::postfix::gateway (
     'smtp_tls_session_cache_database' => {
       'value' => 'btree:/var/lib/postfix/smtp_tls_session_cache'
     },
-    'smtp_use_tls' => {
+    'smtp_send_xforward_command' => {
       'value' => 'yes'
     },
     'smtpd_banner' => {
@@ -211,7 +220,7 @@ class profiles::postfix::gateway (
       'value' => 'check_helo_access hash:/etc/postfix/helo_access, reject_invalid_hostname'
     },
     'smtpd_milters' => {
-      'value' => 'inet:127.0.0.1:33333'
+      'value' => $smtpd_milters
     },
     'smtpd_recipient_restrictions' => {
       'value' => join($smtpd_recipient_restrictions, ', ')
@@ -255,13 +264,10 @@ class profiles::postfix::gateway (
     'smtpd_tls_session_cache_timeout' => {
       'value' => '3600s'
     },
-    'smtpd_use_tls' => {
-      'value' => 'yes'
-    },
     'tls_medium_cipherlist' => {
       'value' => join([
         'ECDSA+AESGCM:ECDH+AESGCM:DH+AESGCM:ECDSA+AES:ECDH+AES:DH+AES',
-        'ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS'
+        'RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS:!3DES'
       ], ':')
     },
     'tls_preempt_cipherlist' => {

site/profiles/manifests/rspamd/edge.pp

@@ -0,0 +1,51 @@
+# local rspamd milter for the mail edge gateways
+# - local_d: /etc/rspamd/local.d file name => content
+# - rules: /etc/rspamd/rules file name => url, re-fetched every puppet run
+class profiles::rspamd::edge (
+  Hash[String, String]           $local_d = {},
+  Hash[String, Stdlib::HTTPSUrl] $rules   = {},
+) {
+
+  package { 'rspamd':
+    ensure => installed,
+  }
+
+  $local_d.each |$name, $content| {
+    file { "/etc/rspamd/local.d/${name}":
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => $content,
+      require => Package['rspamd'],
+      notify  => Service['rspamd'],
+    }
+  }
+
+  file { '/etc/rspamd/rules':
+    ensure  => directory,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    require => Package['rspamd'],
+  }
+
+  $rules.each |$name, $url| {
+    file { "/etc/rspamd/rules/${name}":
+      ensure => file,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0644',
+      source => $url,
+      notify => Service['rspamd'],
+    }
+  }
+
+  # reload keeps the milter answering while config changes apply
+  service { 'rspamd':
+    ensure  => running,
+    enable  => true,
+    restart => '/usr/bin/systemctl reload rspamd',
+    require => Package['rspamd'],
+  }
+}

site/profiles/templates/helpers/certmanager.erb

@@ -1,102 +0,0 @@
-#!<%= @venv_path %>/bin/python
-
-import argparse
-import requests
-import json
-import os
-import yaml
-from zipfile import ZipFile
-
-# remove this after certs are generated everywhere
-requests.packages.urllib3.disable_warnings()
-
-def load_config(config_path):
-    with open(config_path, 'r') as file:
-        config = yaml.safe_load(file)
-    return config['vault']
-
-def authenticate_approle(vault_config):
-    url = f"{vault_config['addr']}/v1/auth/{vault_config['approle_path']}/login"
-    payload = {
-        "role_id": vault_config['role_id'],
-    }
-    response = requests.post(url, json=payload, verify=False)
-    if response.status_code == 200:
-        auth_response = response.json()
-        return auth_response['auth']['client_token']
-    else:
-        print(f"Error authenticating with AppRole: {response.text}")
-        return None
-
-def request_certificate(common_name, alt_names, ip_sans, expiry_days, vault_config):
-    # Authenticate using AppRole and get a token
-    client_token = authenticate_approle(vault_config)
-    if not client_token:
-        print("Failed to authenticate with Vault using AppRole.")
-        return None
-
-    url = f"{vault_config['addr']}/v1/{vault_config['mount_point']}/issue/{vault_config['role_name']}"
-    headers = {'X-Vault-Token': client_token}
-    payload = {
-        "common_name": common_name,
-        "alt_names": ",".join(alt_names),
-        "ip_sans": ",".join(ip_sans),
-        "ttl": f"{expiry_days}d"
-    }
-    response = requests.post(url, headers=headers, json=payload, verify=False)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        print(f"Error requesting certificate: {response.text}")
-        return None
-
-def save_cert_files(certificate_response, common_name, compress, config, json_output):
-    base_path = config.get('output_path', '.')
-    cert_dir = os.path.join(base_path, common_name)
-    if json_output:
-        import json
-        output = {
-            'certificate': certificate_response['data']['certificate'],
-            'private_key': certificate_response['data']['private_key'],
-            'full_chain': certificate_response['data']['issuing_ca'] + "\n" + certificate_response['data']['certificate'],
-        }
-        print(json.dumps(output))
-    elif not compress:
-        os.makedirs(cert_dir, exist_ok=True)
-        with open(os.path.join(cert_dir, "certificate.crt"), "w") as cert_file:
-            cert_file.write(certificate_response['data']['certificate'])
-        with open(os.path.join(cert_dir, "private.key"), "w") as key_file:
-            key_file.write(certificate_response['data']['private_key'])
-        with open(os.path.join(cert_dir, "full_chain.crt"), "w") as full_chain_file:
-            full_chain_file.write(certificate_response['data']['issuing_ca'] + "\n" + certificate_response['data']['certificate'])
-    else:
-        zip_name = f"{os.path.join(base_path, common_name)}.zip"
-        with ZipFile(zip_name, 'w') as zipf:
-            zipf.writestr("certificate.crt", certificate_response['data']['certificate'])
-            zipf.writestr("private.key", certificate_response['data']['private_key'])
-            zipf.writestr("full_chain.crt", certificate_response['data']['issuing_ca'] + "\n" + certificate_response['data']['certificate'])
-
-def main(config_file):
-    config = load_config(config_file)
-    parser = argparse.ArgumentParser(description='Request and retrieve a certificate from Vault.')
-    parser.add_argument('common_name', type=str, help='Common Name for the certificate')
-    parser.add_argument('-a', '--alt-names', type=str, default='', help='Comma-separated alternative names for the certificate')
-    parser.add_argument('-i', '--ip-sans', type=str, default='', help='Comma-separated IP Subject Alternative Names for the certificate')
-    parser.add_argument('-e', '--expiry-days', type=int, default=365, help='Validity of the certificate in days (default: 365)')
-    parser.add_argument('-c', '--compress', action='store_true', help='Compress the certificate, key, and full chain into a zip file')
-    parser.add_argument('--json', action='store_true', help='Output results in JSON format')
-    args = parser.parse_args()
-    alt_names = [name.strip() for name in args.alt_names.split(',') if name]
-    ip_sans = [ip.strip() for ip in args.ip_sans.split(',') if ip]
-    certificate_response = request_certificate(args.common_name, alt_names, ip_sans, args.expiry_days, config)
-    if certificate_response:
-        if args.json:
-            save_cert_files(certificate_response, args.common_name, args.compress, config, True)
-        else:
-            save_cert_files(certificate_response, args.common_name, args.compress, config, False)
-    else:
-        print("Failed to obtain certificate.")
-
-if __name__ == "__main__":
-    config_file = '<%= @config_path %>'
-    main(config_file)

site/profiles/templates/helpers/certmanager_config.yaml.erb

@@ -4,4 +4,4 @@ vault:
   approle_path: '<%= @vault_config['approle_path'] %>'
   mount_point: '<%= @vault_config['mount_point'] %>'
   role_name: '<%= @vault_config['role_name'] %>'
-output_path: '<%= @vault_config['output_path'] %>'
+  output_path: '<%= @vault_config['output_path'] %>'

site/profiles/templates/helpers/sshsignhost.erb

@@ -1,83 +0,0 @@
-#!<%= @venv_path %>/bin/python
-import argparse
-import requests
-import json
-import yaml
-
-# remove this after certs are generated everywhere
-requests.packages.urllib3.disable_warnings()
-
-def load_config(config_path):
-    with open(config_path, 'r') as file:
-        config = yaml.safe_load(file)
-    return config['vault']
-
-def authenticate_approle(vault_config):
-    url = f"{vault_config['addr']}/v1/auth/{vault_config['approle_path']}/login"
-    payload = {
-        "role_id": vault_config['role_id'],
-    }
-    response = requests.post(url, json=payload, verify=False)
-    if response.status_code == 200:
-        auth_response = response.json()
-        return auth_response['auth']['client_token']
-    else:
-        print(f"Error authenticating with AppRole: {response.text}")
-        return None
-
-def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
-    # Authenticate using AppRole and get a token
-    client_token = authenticate_approle(vault_config)
-    if not client_token:
-        print("Failed to authenticate with Vault using AppRole.")
-        return None
-
-    # Prepare the SSH certificate signing request
-    url = f"{vault_config['addr']}/v1/{vault_config['mount_point']}/sign/{vault_config['role_name']}"
-    headers = {'X-Vault-Token': client_token}
-    payload = {
-        "cert_type": "host",
-        "public_key": public_key,
-        "valid_principals": valid_principals,
-        "ttl": ttl
-    }
-
-    # Request the SSH certificate signing
-    response = requests.post(url, headers=headers, json=payload, verify=False)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        print(f"Error requesting certificate: {response.text}")
-        return None
-
-def main(config_file):
-    config = load_config(config_file)
-    parser = argparse.ArgumentParser(description='Sign SSH host certificate using Vault.')
-    parser.add_argument('--public_key', required=True, help='SSH public key as a string')
-    parser.add_argument('--valid_principals', required=True, help='Comma-separated list of valid principals')
-    parser.add_argument('--ttl', default='87600h', help='Time-to-live for the certificate (default: 87600h)')
-    parser.add_argument('--json', action='store_true', help='Output the resulting certificate as JSON')
-
-    args = parser.parse_args()
-
-    # Load configuration
-    config = load_config(config_file)
-
-    # Sign SSH certificate
-    response = sign_ssh_certificate(config, args.public_key, args.valid_principals, args.ttl)
-
-    if response and 'data' in response and 'signed_key' in response['data']:
-        if args.json:
-            output = {
-                'signed_key': response['data']['signed_key'],
-            }
-            print(json.dumps(output))
-        else:
-            print(response['data']['signed_key'])
-    else:
-        print("Error: The response does not contain the expected data.")
-        exit(1)
-
-if __name__ == "__main__":
-    config_file = '<%= @config_path %>'
-    main(config_file)

site/profiles/templates/helpers/sshsignhost_config.yaml.erb

@@ -4,4 +4,3 @@ vault:
   approle_path: '<%= @vault_config['approle_path'] %>'
   mount_point: '<%= @vault_config['mount_point'] %>'
   role_name: '<%= @vault_config['role_name'] %>'
-output_path: '<%= @vault_config['output_path'] %>'

site/roles/manifests/infra/mail/gateway.pp

@@ -6,6 +6,11 @@ class roles::infra::mail::gateway {
   }else{
     include profiles::defaults
     include profiles::base
+    include dnsmasq
+    include profiles::rspamd::edge
     include profiles::postfix::gateway
+
+    # resolv.conf points at dnsmasq, so it must be running first
+    Class['dnsmasq'] -> Class['profiles::dns::resolvconf']
   }
 }

site/roles/manifests/infra/network/router.pp

@@ -0,0 +1,12 @@
+# roles::infra::network::router
+# an ospf router; frr only, interfaces and firewall are managed outside puppet
+#
+class roles::infra::network::router {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}