Migrate consul servers to puppet-on-k8s

Migrated nodes have no facts or exported resources in the k8s PuppetDB until their next agent run (up to splay + runinterval), which breaks exported-resource discovery for services migrated as a group. A noop run registers both at compile time.

- add a refreshonly exec in profiles::puppet::migrate, subscribed to puppet.conf, that launches a detached noop agent run via systemd-run
- skip the exec while a previous puppet-migrate-noop unit is still active

Reviewed-on: #561
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

hieradata/nodes/aubne1nxvm2133.main.unkin.net.yaml

@@ -1,24 +0,0 @@
----
-networking_loopback0_ip: 198.18.2.162
-networking_wg_link: 198.18.2.178/31
-networking_vpc_cidr: 172.18.32.0/24
-
-# wg0 p2p to prodnxsr0020; lab routes stay pinned to wg0 sourced from the /31 until ospf learns them
-wireguard::interfaces:
-  wg0:
-    addresses:
-      - 198.18.2.179/31
-    listen_port: 51820
-    table: 'off'
-    post_up:
-      - ip route replace 198.18.0.0/16 dev wg0 src 198.18.2.179
-      - ip route replace 10.10.12.200/32 dev wg0 src 198.18.2.179
-    private_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAe680Uimksc8igs5HhN58HI1g4WsneHsI3Az66pAZIP7AQaISMCuQD51PGGzFxZqBd3wNf5eyhZdyBV3Yl5Rb+1lte28gld1gYtoTRkLW64BZOGcjs4kaIgIfKjPpp0pcdEBzFb0nVeMRkjvJFw3BGaj05yN+eUCNMf3qYpP47SD4JE8DnpgtRP09FhZ27suJylkB2SkrX+MUJPnuZ3SrKLYEI8kCsXB6puCMRALWAysjoWEXyElT+tShRv5eOcGvcUIenygBVTgBTyu7EsLtDtTmY0gGzG3VMIXxriW0RQ4UwcAW0fkiUxWc+KdVKysVthyjjuUupL6we0GTqC3KSjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB7FdQ73iu+fttRMUQwmaH5gDBpnsnowpVZP96vIs6ldfkm0mtO4ymnU6e+D3U9CRayiZXdhGVfHmHNb8L3s98YsQE=]
-    peers:
-      - public_key: 'mPiOahdhI4jyS7IpkSW1HV7ERd8VH53yv/H4hCxES1g='
-        endpoint: 103.216.191.185:51822
-        allowed_ips:
-          - 224.0.0.5/32
-          - 198.18.0.0/16
-          - 10.10.12.200/32
-        persistent_keepalive: 25

hieradata/nodes/ausyd2nxvm2131.main.unkin.net.yaml

@@ -1,24 +0,0 @@
----
-networking_loopback0_ip: 198.18.2.161
-networking_wg_link: 198.18.2.176/31
-networking_vpc_cidr: 172.18.30.0/24
-
-# wg0 p2p to prodnxsr0020; lab routes stay pinned to wg0 sourced from the /31 until ospf learns them
-wireguard::interfaces:
-  wg0:
-    addresses:
-      - 198.18.2.177/31
-    listen_port: 51820
-    table: 'off'
-    post_up:
-      - ip route replace 198.18.0.0/16 dev wg0 src 198.18.2.177
-      - ip route replace 10.10.12.200/32 dev wg0 src 198.18.2.177
-    private_key: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAEFLiAUx3WWC+nAl+XfKayu1J9gXlQI3dGAx6/0NCgIf/gDk64lrnuwzCJW9W0y00utkhoqdydx0rvjfFlI5v5+3vKWb/zyPQwcroQ1T/Tp+5627F6skuNa5LMVNk+GQHb729JluUmCW2Q8Rj4BSYYQKBS7tftgAnBiVB3aXANxgBeBZuONbI8hhFjTXHrSvDidgIHJzMpWwU40ZAwhk6B4Z9bTQ0yhs8Ucq6klaF2lDM6OO3mCrBeylCs9mgoLTE2q5B/+EqCHssNpQdhdKJ3FXp6IISn5Q3nnr7px7ks1Gzl5Ckke1D5O0WfXa1QvHQsiTkMF8B7tkY2IFHcfQz1DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBA5YvFDxQEIxXzHJr2fVYmjgDBgflL2FY/Eu+thiGsS11m7VZ6qCy+x0d/8lcIa95aS7F3PadIyo4xGMgKtfQNzHwI=]
-    peers:
-      - public_key: 'XB2X5NwuoE+QJLjZG4i/0+6Jj0AtfK+UsZZ7wEfYLjk='
-        endpoint: 103.216.191.185:51821
-        allowed_ips:
-          - 224.0.0.5/32
-          - 198.18.0.0/16
-          - 10.10.12.200/32
-        persistent_keepalive: 25

hieradata/roles/infra/network/wgvpn.yaml

@@ -1,42 +0,0 @@
----
-hiera_include:
-  - wireguard
-  - frrouting
-
-# no consul datacentre at the edge, so nothing to register or scrape via consul
-profiles::consul::client::enable: false
-exporters::node_exporter::enable: false
-
-# ssh only from the lab over the vpn
-ssh::server::options:
-  AllowUsers: '*@198.18.0.0/16 *@10.10.12.200/32'
-
-# routing
-sysctl::base::values:
-  net.ipv4.ip_forward:
-    value: '1'
-
-# frrouting: p2p ospf over wg0; loopback /32, wg /31 and vpc /24 only, no default route in or out
-frrouting::daemons:
-  ospfd: true
-frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
-frrouting::ospfd_interfaces:
-  wg0:
-    area: 0.0.0.0
-    network: point-to-point
-  dum0:
-    area: 0.0.0.0
-    passive: true
-frrouting::ospfd_network_interfaces:
-  "%{hiera('networking_vpc_cidr')}":
-    area: 0.0.0.0
-    passive: true
-frrouting::ospfd_redistribute:
-  - connected
-frrouting::ospfd_redistribute_connected_permit:
-  - "%{hiera('networking_loopback0_ip')}/32"
-  - "%{hiera('networking_wg_link')}"
-  - "%{hiera('networking_vpc_cidr')}"
-frrouting::ospfd_deny_default_in: true
-frrouting::ospf_preferred_source_enable: true
-frrouting::ospf_preferred_source: "%{hiera('networking_loopback0_ip')}"

hieradata/roles/infra/storage/consul.yaml

@@ -3,6 +3,8 @@ hiera_include:
   - frrouting
   - exporters::frr_exporter
 
+profiles::puppet::migrate::enabled: true
+
 profiles::consul::server::members_lookup: true
 profiles::consul::server::data_dir: /data/consul
 profiles::consul::server::addresses:

modules/frrouting/manifests/init.pp

@@ -6,13 +6,10 @@ class frrouting (
   String  $service_name             = 'frr',
   Hash    $daemons                  = {},
   Hash    $ospfd_interfaces         = {},
-  Hash[Stdlib::IP::Address::V4::CIDR, Hash] $ospfd_network_interfaces = {}, # like ospfd_interfaces, keyed by a subnet the interface holds
   String  $ospfd_router_id          = $facts['networking']['ip'],
   Array[String] $ospfd_redistribute = [],
   Array[String] $ospfd_networks     = [],
   Boolean $ospfd_default_originate_always = false,
-  Array[Stdlib::IP::Address::V4::CIDR] $ospfd_redistribute_connected_permit = [], # only these exact connected prefixes are redistributed
-  Boolean $ospfd_deny_default_in    = false, # keep a learned default route out of the routing table
   Boolean $mpls_te_enabled                   = false,
   Optional[String] $mpls_ldp_router_id       = undef,
   Optional[String] $mpls_ldp_transport_addr  = undef,
@@ -29,17 +26,6 @@ class frrouting (
     warning('frrouting: ospf_exclude_k8s_enable is true but no k8s_*_cidr provided; nothing will be filtered.')
   }
 
-  $network_interfaces = $ospfd_network_interfaces.reduce({}) |$memo, $entry| {
-    $matched = $facts['networking']['interfaces'].filter |$name, $iface| {
-      $iface['bindings'] =~ Array and $iface['bindings'].any |$binding| { stdlib::ip_in_range($binding['address'], $entry[0]) }
-    }
-    if empty($matched) {
-      fail("frrouting: no interface holds an address in ${entry[0]}")
-    }
-    $memo + $matched.keys.reduce({}) |$acc, $name| { $acc + { $name => $entry[1] } }
-  }
-  $ospfd_interfaces_all = $ospfd_interfaces + $network_interfaces
-
   $daemons_defaults = {
     'bgpd'    => false,
     'ospfd'   => true,

modules/frrouting/templates/frr.conf.erb

@@ -3,14 +3,11 @@ frr defaults traditional
 hostname <%= @hostname %>
 no ipv6 forwarding
 
-<% @ospfd_interfaces_all.each do |iface, params| -%>
+<% @ospfd_interfaces.each do |iface, params| -%>
 interface <%= iface %>
 <% if params['area'] -%>
  ip ospf area <%= params['area'] %>
 <% end -%>
-<% if params['network'] -%>
- ip ospf network <%= params['network'] %>
-<% end -%>
 <% if params['passive'] == true -%>
  ip ospf passive
 <% end -%>
@@ -31,27 +28,12 @@ route-map CONNECTED-NON-K8S permit 100
 exit
 <% end -%>
 
-<% if @ospfd_redistribute_connected_permit.any? -%>
-<% @ospfd_redistribute_connected_permit.each_with_index do |prefix, i| -%>
-ip prefix-list CONNECTED-PERMIT seq <%= (i + 1) * 5 %> permit <%= prefix %>
-<% end -%>
-route-map CONNECTED-PERMIT permit 10
- match ip address prefix-list CONNECTED-PERMIT
-exit
-<% end -%>
-<% if @ospfd_deny_default_in -%>
-access-list NO-DEFAULT seq 5 deny 0.0.0.0/0 exact-match
-access-list NO-DEFAULT seq 10 permit any
-<% end -%>
-
 router ospf
  ospf router-id <%= @ospfd_router_id %>
  log-adjacency-changes detail
 <% @ospfd_redistribute.each do |type| -%>
 <% if @ospf_exclude_k8s_enable && type == 'connected' && (@k8s_cluster_cidr || @k8s_service_cidr) -%>
  redistribute connected route-map CONNECTED-NON-K8S
-<% elsif type == 'connected' && @ospfd_redistribute_connected_permit.any? -%>
- redistribute connected route-map CONNECTED-PERMIT
 <% else -%>
  redistribute <%= type %>
 <% end -%>
@@ -59,9 +41,6 @@ router ospf
 <% @ospfd_networks.each do |network| -%>
  network <%= network %>
 <% end -%>
-<% if @ospfd_deny_default_in -%>
- distribute-list NO-DEFAULT in
-<% end -%>
 <% if @ospfd_default_originate_always -%>
  default-information originate always
 <% end -%>

modules/wireguard/manifests/interface.pp

@@ -4,8 +4,6 @@ define wireguard::interface (
   Optional[Stdlib::Port]               $listen_port = undef,
   Optional[Integer[1280, 9000]]        $mtu         = undef,
   Optional[Sensitive[String[1]]]       $private_key = undef,
-  Optional[String[1]]                  $table       = undef,
-  Array[String[1]]                     $post_up     = [],
   Array[Struct[{
     public_key           => String[1],
     allowed_ips          => Variant[String[1], Array[String[1], 1]],

modules/wireguard/templates/wg.conf.erb

@@ -9,17 +9,11 @@ ListenPort = <%= @listen_port %>
 <% if @mtu -%>
 MTU = <%= @mtu %>
 <% end -%>
-<% if @table -%>
-Table = <%= @table %>
-<% end -%>
 <% if @key -%>
 PrivateKey = <%= @key %>
 <% else -%>
 PostUp = wg set %i private-key /etc/wireguard/%i.key
 <% end -%>
-<% @post_up.each do |cmd| -%>
-PostUp = <%= cmd %>
-<% end -%>
 <% @peers.each do |peer| -%>
 
 [Peer]

site/profiles/manifests/puppet/migrate.pp

@@ -17,6 +17,8 @@
 # still points at it when it applies this change). profiles::puppet::client
 # rewrites puppet.conf with the new server / ca_server / report_server / ssldir
 # and notifies Service['puppet'], so the very next agent run lands on k8s.
+# The puppet.conf change also fires a detached noop run so facts and exported
+# resources register in the k8s PuppetDB immediately.
 #
 # NOTE: this is deliberately hiera-driven per node/role so waves can be rolled
 # out and rolled back one target at a time. The class only manages the fresh
@@ -83,5 +85,15 @@ class profiles::puppet::migrate (
       group  => 'root',
       mode   => '0771',
     }
+
+    exec { 'puppet-migrate-noop-run':
+      command     => join([
+        '/usr/bin/systemd-run --unit=puppet-migrate-noop --collect',
+        '/opt/puppetlabs/bin/puppet agent --onetime --no-daemonize --noop --no-splay --waitforlock 10 --maxwaitforlock 900',
+      ], ' '),
+      unless      => '/usr/bin/systemctl is-active --quiet puppet-migrate-noop',
+      refreshonly => true,
+      subscribe   => File['/etc/puppetlabs/puppet/puppet.conf'],
+    }
   }
 }

site/roles/manifests/infra/network/wgvpn.pp

@@ -1,12 +0,0 @@
-# roles::infra::network::wgvpn
-# a wireguard vpn edge host; routes its vpc to the lab over wg0 with ospf
-#
-class roles::infra::network::wgvpn {
-  if $facts['firstrun'] {
-    include profiles::defaults
-    include profiles::firstrun::init
-  }else{
-    include profiles::defaults
-    include profiles::base
-  }
-}