feat: add nomad agent role (#206)

- add nomad agent role
- mount cephfs volume nomadfs to /shared/nomad
- manage docker volume path to be /shared/nomad

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/206

hieradata/nodes/ausyd1nxvm1064.main.unkin.net.yaml

@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.74
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.74
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
+    gateway: 198.18.13.254
+docker::bip: '198.18.64.254/24'

hieradata/nodes/ausyd1nxvm1065.main.unkin.net.yaml

@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.75
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.75
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
+    gateway: 198.18.13.254
+docker::bip: '198.18.65.254/24'

hieradata/nodes/ausyd1nxvm1066.main.unkin.net.yaml

@@ -2,6 +2,14 @@
 networking::interfaces:
   eth0:
     ipaddress: 198.18.13.76
+  ens19:
+    ensure: present
+    family: inet
+    method: static
+    ipaddress: 10.18.15.76
+    netmask: 255.255.255.0
+    onboot: true
 networking::routes:
   default:
-    gateway: 198.18.13.254
+    gateway: 198.18.13.254
+docker::bip: '198.18.66.254/24'

hieradata/roles/infra/nomad/agent.eyaml

@@ -0,0 +1,2 @@
+---
+ceph::key::media: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAEzl2zX8ok6iURymPLbvkFE/dZhunzwisNCsLE5Tx6Pmil0PNx7iFULHXxWU6HMVOxzJmgcnTY+NsRnoTMSpXHGeC3wmD525T4xO4wvG/l9apzmPs1NTI4hcyYCj4NkAKyo249xXEodU4uM2AZp3ZgLBLf2cZpOe0/SPngWSq0kC4pJMnxWH+YsQ/O/1EMxpjSgMMna4YcMtcW2M0+wRhASNSTV7icsiAp6F/cInZuP44ASviHmM6+aMEhygBariOT80kaHZZI+aP6vqSd9lChXCV5qjRzeoEBIKOzT2ZBj41RTsF75J8UYFPwY7dp584tDpiIedUSR9IRCJe4d0uTjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCau9oXJiYiCGgvxZl62EsagDDgX2cU3+3QCkImEGS1oBahkPl3fYHKynbRi0ZQW1CoW0UFRzY8FmWmGkowns9OsIM=]

hieradata/roles/infra/nomad/agent.yaml

@@ -0,0 +1,55 @@
+---
+
+hiera_include:
+  - docker
+  - docker::networks
+  - profiles::nomad::node
+
+docker::version: latest
+docker::curl_ensure: false
+docker::root_dir: /data/docker
+docker::bip: '198.18.101.254/24'
+docker::ip_forward: true
+docker::ip_masq: false
+docker::iptables: true
+
+profiles::yum::global::repos:
+  ceph-reef:
+    name: ceph-reef
+    descr: ceph reef repository
+    target: /etc/yum.repos.d/ceph-reef.repo
+    baseurl: https://edgecache.query.consul/ceph/yum/el%{facts.os.release.major}/%{facts.os.architecture}
+    gpgcheck: 0,
+    mirrorlist: absent
+
+profiles::ceph::client::keyrings:
+  nomad:
+    key: "%{hiera('ceph::key::media')}"
+
+profiles::packages::include:
+  nomad: {}
+
+profiles::nomad::node::client: true
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - client.global.nomad
+  - client.au-syd1.nomad
+  - nomad-client.service.consul
+  - nomad-client.query.consul
+  - "nomad-client.service.%{facts.country}-%{facts.region}.consul"
+
+# configure consul service
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: nomad-client
+    disposition: write
+  - resource: agent_prefix
+    segment: ''
+    disposition: read
+  - resource: node_prefix
+    segment: ''
+    disposition: write
+  - resource: service_prefix
+    segment: ''
+    disposition: write

site/profiles/manifests/nomad/node.pp

@@ -1,6 +1,7 @@
 # profiles::nomad::node
 class profiles::nomad::node (
-  Stdlib::Absolutepath $data_dir = '/data/nomad',
+  Stdlib::Absolutepath $data_dir    = '/data/nomad',
+  Stdlib::Absolutepath $nomad_root  = '/shared/nomad',
   Integer $bootstrap_expect = 3,
   Boolean $server = false,
   Boolean $client = false,
@@ -30,6 +31,20 @@ class profiles::nomad::node (
     }
   }
 
+  if $client {
+
+    include profiles::ceph::client
+
+    # manage the sharedvol
+    profiles::storage::cephfsvol {"${::facts['networking']['fqdn']}_nomad":
+      mount       => $nomad_root,
+      keyring     => '/etc/ceph/ceph.client.nomad.keyring',
+      cephfs_name => 'nomad',
+      cephfs_fs   => 'nomadfs',
+      require     => Profiles::Ceph::Keyring['nomad'],
+    }
+  }
+
   file { $data_dir:
     ensure  => directory,
     owner   => $user,

site/profiles/templates/nomad/config.hcl.erb

@@ -36,4 +36,3 @@ tls {
   verify_server_hostname = true
   verify_https_client    = false
 }
-

site/roles/manifests/infra/nomad/agent.pp

@@ -0,0 +1,11 @@
+# a role to deploy a nomad agent
+class roles::infra::nomad::agent {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+    include profiles::base::datavol
+  }
+}