From c225564bdb408e7360f500703270809f1123c2ba Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Sun, 6 Apr 2025 16:38:04 +1000
Subject: [PATCH] feat: continue incus implementation (#245)

- migrate to systemd-networkd
- setup dummy, bridge and static/ethernet interfaces
- manage sshd.service droping to start ssh after networking is online
- enable ip forewarding
- add fastpool/data/incus dataset
- enable ospf and frr
- add loopback0 as ssh listenaddress
- add loopback1/2 for ceph cluster/public traffic

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/245
---
 .../nodes/prodnxsr0009.main.unkin.net.yaml    | 21 +++---
 .../nodes/prodnxsr0010.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0011.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0012.main.unkin.net.yaml    | 13 ++--
 .../nodes/prodnxsr0013.main.unkin.net.yaml    | 13 ++--
 hieradata/roles/infra/incus/node.yaml         | 70 ++++++++++++++++++
 modules/libs/lib/facter/subnet_facts.rb       | 13 +++-
 modules/networking/manifests/bridge.pp        | 22 ++++++
 modules/networking/manifests/dummy.pp         | 18 +++++
 modules/networking/manifests/init.pp          | 72 +++++++++++++------
 modules/networking/manifests/static.pp        | 26 +++++++
 .../networking/templates/bridge.netdev.erb    |  3 +
 modules/networking/templates/dummy.netdev.erb |  3 +
 .../networking/templates/networkd-link.erb    |  8 +++
 .../networking/templates/networkd-network.erb | 37 ++++++++++
 site/profiles/manifests/base.pp               |  1 +
 site/profiles/manifests/ssh/service.pp        | 15 ++++
 17 files changed, 310 insertions(+), 51 deletions(-)
 create mode 100644 modules/networking/manifests/bridge.pp
 create mode 100644 modules/networking/manifests/dummy.pp
 create mode 100644 modules/networking/manifests/static.pp
 create mode 100644 modules/networking/templates/bridge.netdev.erb
 create mode 100644 modules/networking/templates/dummy.netdev.erb
 create mode 100644 modules/networking/templates/networkd-link.erb
 create mode 100644 modules/networking/templates/networkd-network.erb
 create mode 100644 site/profiles/manifests/ssh/service.pp

diff --git a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
index 8ae6946..a1cc562 100644
--- a/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0009.main.unkin.net.yaml
@@ -1,15 +1,18 @@
 ---
+networking_loopback0_ip: 198.18.19.9  # management loopback
+networking_loopback1_ip: 198.18.22.9  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.9  # ceph-public loopback
+networking_br10_ip: 198.18.25.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:8d
     ipaddress: 198.18.15.9
-  enp3s0:
-    ipaddress: 10.18.15.9
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:5d
+    ipaddress: 198.18.21.9
 
-zfs::zpools:
-  fastpool:
-    ensure: present
-    disk: /dev/nvme0n1
+#zfs::zpools:
+#  fastpool:
+#    ensure: present
+#    disk: /dev/nvme0n1
diff --git a/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
index d9e7592..a7e0f91 100644
--- a/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0010.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.10  # management loopback
+networking_loopback1_ip: 198.18.22.10  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.10  # ceph-public loopback
+networking_br10_ip: 198.18.26.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:37
     ipaddress: 198.18.15.10
-  enp3s0:
-    ipaddress: 10.18.15.10
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:de
+    ipaddress: 198.18.21.10
diff --git a/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
index 28a45ab..e146b42 100644
--- a/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0011.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.11  # management loopback
+networking_loopback1_ip: 198.18.22.11  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.11  # ceph-public loopback
+networking_br10_ip: 198.18.27.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:38:e9:0f
     ipaddress: 198.18.15.11
-  enp3s0:
-    ipaddress: 10.18.15.11
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:55
+    ipaddress: 198.18.21.11
diff --git a/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
index fbb68c4..c309a59 100644
--- a/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0012.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.12  # management loopback
+networking_loopback1_ip: 198.18.22.12  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.12  # ceph-public loopback
+networking_br10_ip: 198.18.28.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:4f:05:1e
     ipaddress: 198.18.15.12
-  enp3s0:
-    ipaddress: 10.18.15.12
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:e5
+    ipaddress: 198.18.21.12
diff --git a/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
index 221e494..86221c3 100644
--- a/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
+++ b/hieradata/nodes/prodnxsr0013.main.unkin.net.yaml
@@ -1,10 +1,13 @@
 ---
+networking_loopback0_ip: 198.18.19.13  # management loopback
+networking_loopback1_ip: 198.18.22.13  # ceph-cluster loopback
+networking_loopback2_ip: 198.18.23.13  # ceph-public loopback
+networking_br10_ip: 198.18.29.254
 networking::interfaces:
   enp2s0:
+    mac: 70:b5:e8:4f:04:b0
     ipaddress: 198.18.15.13
-  enp3s0:
-    ipaddress: 10.18.15.13
-    mtu: 9000
-networking::routes:
-  default:
     gateway: 198.18.15.254
+  enp3s0:
+    mac: 00:e0:4c:68:0f:36
+    ipaddress: 198.18.21.13
diff --git a/hieradata/roles/infra/incus/node.yaml b/hieradata/roles/infra/incus/node.yaml
index 229a047..720285e 100644
--- a/hieradata/roles/infra/incus/node.yaml
+++ b/hieradata/roles/infra/incus/node.yaml
@@ -1,8 +1,12 @@
 ---
 hiera_include:
+  - frrouting
   - incus
   - zfs
 
+profiles::packages::include:
+  bridge-utils: {}
+
 profiles::pki::vault::alt_names:
   - incus.service.consul
   - incus.query.consul
@@ -46,6 +50,65 @@ profiles::yum::global::repos:
     gpgkey: https://packagerepo.service.consul/zfs/rhel9/kmod-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-openzfs-2022
     mirrorlist: absent
 
+# networking
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+#networking::use_networkd: true
+networking::interfaces:
+  enp2s0:
+    type: physical
+    txqueuelen: 10000
+    forwarding: true
+  enp3s0:
+    type: physical
+    mtu: 9000
+    txqueuelen: 10000
+    forwarding: true
+  loopback0:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback0_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  loopback1:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback1_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  loopback2:
+    type: dummy
+    ipaddress: "%{hiera('networking_loopback2_ip')}"
+    netmask: 255.255.255.255
+    mtu: 9000
+  br10:
+    type: bridge
+    bridge: true
+    ipaddress: "%{hiera('networking_br10_ip')}"
+    netmask: 255.255.255.0
+    nocarrier: true
+    forwarding: true
+
+# frrouting
+frrouting::ospfd_router_id: "%{hiera('networking_loopback0_ip')}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  enp2s0:
+    area: 0.0.0.0
+  enp3s0:
+    area: 0.0.0.0
+  loopback0:
+    area: 0.0.0.0
+  loopback1:
+    area: 0.0.0.0
+  loopback2:
+    area: 0.0.0.0
+  br10:
+    area: 0.0.0.0
+
+# add loopback interfaces to ssh list
+ssh::server::options:
+  ListenAddress:
+    - "%{hiera('networking_loopback0_ip')}"
 
 # zfs settings
 zfs::manage_repo: false
@@ -67,6 +130,9 @@ zfs::datasets:
   fastpool/data:
     canmount: 'on'
     mountpoint: '/data'
+  fastpool/data/incus:
+    canmount: 'on'
+    mountpoint: '/data/incus'
 
 # manage incus
 incus::cluster::members_lookup: true
@@ -101,6 +167,10 @@ sysctl::base::values:
     value: '8192'
   vm.max_map_count:
     value: '262144'
+  net.ipv4.conf.all.forwarding:
+    value: '1'
+  net.ipv6.conf.all.forwarding:
+    value: '1'
 
 # limits.d recommendations
 limits::entries:
diff --git a/modules/libs/lib/facter/subnet_facts.rb b/modules/libs/lib/facter/subnet_facts.rb
index 458c8e0..28325fd 100644
--- a/modules/libs/lib/facter/subnet_facts.rb
+++ b/modules/libs/lib/facter/subnet_facts.rb
@@ -10,7 +10,18 @@ class SubnetAttributes
     '198.18.15.0/24' => { environment: 'prod', region: 'syd1', country: 'au' },
     '198.18.16.0/24' => { environment: 'test', region: 'syd1', country: 'au' },
     '198.18.17.0/24' => { environment: 'prod', region: 'drw1', country: 'au' },
-    '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' }
+    '198.18.18.0/24' => { environment: 'test', region: 'drw1', country: 'au' },
+    '198.18.19.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # loopbacks
+    '198.18.20.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # RESERVED
+    '198.18.21.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # physical network 2.5gbe
+    '198.18.22.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph cluster
+    '198.18.23.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # ceph public
+    '198.18.24.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # dmz 1
+    '198.18.25.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0009
+    '198.18.26.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0010
+    '198.18.27.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0011
+    '198.18.28.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }, # common node0012
+    '198.18.29.0/24' => { environment: 'prod', region: 'syd1', country: 'au' }  # common node0013
   }.freeze
 
   # Default attributes if no subnet matches, also defined as a constant
diff --git a/modules/networking/manifests/bridge.pp b/modules/networking/manifests/bridge.pp
new file mode 100644
index 0000000..7fd30f1
--- /dev/null
+++ b/modules/networking/manifests/bridge.pp
@@ -0,0 +1,22 @@
+# manage bridges and bridge slaves
+define networking::bridge (
+  String                $type,
+  Optional[Stdlib::IP::Address] $ipaddress,
+  Optional[Stdlib::IP::Address] $netmask   = undef,
+  Optional[Stdlib::IP::Address] $gateway   = undef,
+  Optional[Boolean]             $nocarrier = undef,
+  Boolean               $bridge = true,
+  Integer[100-9200]     $mtu    = 1500,
+  Optional[Boolean]     $forwarding = false,
+) {
+  include systemd
+
+  systemd::network { "${title}.netdev":
+    content => template('networking/bridge.netdev.erb'),
+  }
+
+  # Use shared template, it will detect bridge=true and skip Address/DNS/etc
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+}
diff --git a/modules/networking/manifests/dummy.pp b/modules/networking/manifests/dummy.pp
new file mode 100644
index 0000000..cae8e10
--- /dev/null
+++ b/modules/networking/manifests/dummy.pp
@@ -0,0 +1,18 @@
+# manage dummy/loopback interfaces
+define networking::dummy (
+  String                $type,
+  Stdlib::IP::Address   $ipaddress,
+  Stdlib::IP::Address   $netmask,
+  Integer[100-9200]     $mtu = 1500,
+  Optional[Boolean]     $forwarding = false,
+) {
+  include systemd
+
+  systemd::network { "${title}.netdev":
+    content => template('networking/dummy.netdev.erb'),
+  }
+
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+}
diff --git a/modules/networking/manifests/init.pp b/modules/networking/manifests/init.pp
index d2076ca..8a28690 100644
--- a/modules/networking/manifests/init.pp
+++ b/modules/networking/manifests/init.pp
@@ -4,37 +4,67 @@ class networking (
   Hash $interface_defaults = {},
   Hash $routes = {},
   Hash $route_defaults = {},
+  Boolean $use_networkd = lookup('systemd::manage_networkd'),
 ){
 
   include network
   include networking::params
 
-  # manage interfaces
-  $interfaces.each | $interface, $data | {
-    $merged_data = merge($interface_defaults, $data)
-    network_config { $interface:
-      *      => $merged_data,
-      notify => Exec['networking_reload_network'],
-    }
-  }
+  if $use_networkd {
 
-  # manage routes
-  $routes.each | $route, $data | {
-    $merged_data = merge($route_defaults, $data)
-    network_route { $route:
-      *      => $merged_data,
-      notify => Exec['networking_reload_network'],
+    include systemd
+
+    service { 'NetworkManager':
+      ensure => 'stopped',
+      enable => false,
+    }
+
+    $interfaces.each |String $iface, Hash $data| {
+      $type   = $data['type']
+      #$params = $data.filter |$key, $value| { $key != 'type' }
+
+      case $type {
+        'bridge': { networking::bridge { $iface: * => $data } }
+        'dummy': { networking::dummy { $iface: * => $data } }
+        'static': { networking::static { $iface: * => $data } }
+        'physical': { networking::static { $iface: * => $data } }
+        default: {
+          fail("Unsupported interface type '${type}' for interface '${iface}'")
+        }
+      }
+    }
+  }else{
+    # manage interfaces
+    $interfaces.each | $interface, $data | {
+      $merged_data = merge($interface_defaults, $data)
+      network_config { $interface:
+        *      => $merged_data,
+        notify => Exec['networking_reload_network'],
+      }
+    }
+
+    # manage routes
+    $routes.each | $route, $data | {
+      $merged_data = merge($route_defaults, $data)
+      network_route { $route:
+        *      => $merged_data,
+        notify => Exec['networking_reload_network'],
+      }
     }
   }
 
   # determine which networking service to restart
-  $restart_command = $facts['os']['family'] ? {
-    'RedHat' => $facts['os']['release']['major'] ? {
-      '8' => '/usr/bin/systemctl restart network',
-      '9' => '/usr/bin/systemctl restart NetworkManager',
-    },
-    'Debian' => '/usr/bin/systemctl restart networking',
-    default  => fail('Unsupported OS in networking-restart-command'),
+  $restart_command = $use_networkd ? {
+    true    => '/usr/bin/systemctl restart systemd-networkd',
+    default => $facts['os']['family'] ? {
+      'RedHat' => $facts['os']['release']['major'] ? {
+        '8' => '/usr/bin/systemctl restart network',
+        '9' => '/usr/bin/systemctl restart NetworkManager',
+        default => fail('Unsupported RedHat OS release for networking restart'),
+      },
+      'Debian' => '/usr/bin/systemctl restart networking',
+      default  => fail('Unsupported OS in networking-restart-command'),
+    }
   }
 
   # restart network/networking only if $restart_networking boolean is true
diff --git a/modules/networking/manifests/static.pp b/modules/networking/manifests/static.pp
new file mode 100644
index 0000000..018ef89
--- /dev/null
+++ b/modules/networking/manifests/static.pp
@@ -0,0 +1,26 @@
+# manage static interfaces
+define networking::static (
+  String                $type,
+  Stdlib::IP::Address   $ipaddress,
+  Stdlib::IP::Address   $netmask  = '255.255.255.0',
+  Integer[100-9200]     $mtu      = 1500,
+  Optional[Boolean]     $forwarding = false,
+  Optional[Stdlib::IP::Address]         $gateway = undef,
+  Optional[Array[Stdlib::IP::Address]]  $dns = undef,
+  Optional[Array[Stdlib::Fqdn]]         $domains = undef,
+  Optional[Integer[0-4096]]             $vlan = undef,
+  Optional[Variant[Boolean,String]]     $bridge = undef,
+  Optional[Integer[0-4294967294]]       $txqueuelen = undef,
+  Optional[Stdlib::MAC]                 $mac = undef,
+) {
+  include systemd
+
+  systemd::network { "${title}.network":
+    content => template('networking/networkd-network.erb'),
+  }
+  #if $type == 'physical' and $mac {
+  #  systemd::network { "${title}.link":
+  #    content => template('networking/networkd-link.erb'),
+  #  }
+  #}
+}
diff --git a/modules/networking/templates/bridge.netdev.erb b/modules/networking/templates/bridge.netdev.erb
new file mode 100644
index 0000000..94253e0
--- /dev/null
+++ b/modules/networking/templates/bridge.netdev.erb
@@ -0,0 +1,3 @@
+[NetDev]
+Name=<%= @title %>
+Kind=bridge
diff --git a/modules/networking/templates/dummy.netdev.erb b/modules/networking/templates/dummy.netdev.erb
new file mode 100644
index 0000000..05ef8f4
--- /dev/null
+++ b/modules/networking/templates/dummy.netdev.erb
@@ -0,0 +1,3 @@
+[NetDev]
+Name=<%= @title %>
+Kind=dummy
diff --git a/modules/networking/templates/networkd-link.erb b/modules/networking/templates/networkd-link.erb
new file mode 100644
index 0000000..d45240d
--- /dev/null
+++ b/modules/networking/templates/networkd-link.erb
@@ -0,0 +1,8 @@
+[Match]
+MACAddress=<%= @mac %>
+
+[Link]
+MTUBytes=<%= @mtu %>
+<% if @txqueuelen and @txqueuelen >= 1 -%>
+TransmitQueueLength=<%= @txqueuelen %>
+<% end -%>
diff --git a/modules/networking/templates/networkd-network.erb b/modules/networking/templates/networkd-network.erb
new file mode 100644
index 0000000..298304d
--- /dev/null
+++ b/modules/networking/templates/networkd-network.erb
@@ -0,0 +1,37 @@
+[Match]
+Name=<%= @title %>
+
+[Network]
+<% if @ipaddress && @netmask -%>
+Address=<%= @ipaddress %>/<%= IPAddr.new(@netmask).to_i.to_s(2).count('1') %>
+<% end -%>
+<% if @gateway -%>
+Gateway=<%= @gateway %>
+<% end -%>
+<% if @dns -%>
+DNS=<%= Array(@dns).join(' ') %>
+<% end -%>
+<% if @domains -%>
+Domains=<%= Array(@domains).join(' ') %>
+<% end -%>
+<% if @bridge and @bridge != true -%>
+Bridge=<%= @bridge %>
+<% end -%>
+<% if @vlan -%>
+VLAN=<%= @vlan %>
+<% end -%>
+<% if @nocarrier and @nocarrier == true -%>
+ConfigureWithoutCarrier=true
+DuplicateAddressDetection=none
+RequiredForOnline=no-carrier
+<% end -%>
+<% if @type == 'dummy' -%>
+LinkLocalAddressing=no
+ActivationPolicy=always-up
+<% end -%>
+<% if @forwarding and @forwarding == true -%>
+IPForward=true
+<% end -%>
+
+[Link]
+MTUBytes=<%= @mtu %>
diff --git a/site/profiles/manifests/base.pp b/site/profiles/manifests/base.pp
index c1a4feb..4ec5150 100644
--- a/site/profiles/manifests/base.pp
+++ b/site/profiles/manifests/base.pp
@@ -33,6 +33,7 @@ class profiles::base (
     include profiles::pki::vault
     include profiles::ssh::sign
     include profiles::ssh::knownhosts
+    include profiles::ssh::service
     include profiles::cloudinit::init
     include profiles::metrics::default
     include profiles::helpers::node_lookup
diff --git a/site/profiles/manifests/ssh/service.pp b/site/profiles/manifests/ssh/service.pp
new file mode 100644
index 0000000..e334016
--- /dev/null
+++ b/site/profiles/manifests/ssh/service.pp
@@ -0,0 +1,15 @@
+# profiles::ssh::service
+# saz-ssh manages the service, this is just some additional stuff
+class profiles::ssh::service {
+
+  # set sshd to start
+  systemd::manage_dropin { 'after-network-online.conf':
+    ensure     => present,
+    unit       => 'sshd.service',
+    unit_entry => {
+      'After' => [
+        'network-online.target',
+      ],
+    },
+  }
+}