feat: add haproxy2 role (#322)

- add basic haproxy2 role
- add peers and resolvers
- add haproxy2+ metrics frontend

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/322

hieradata/country/au/region/syd1/infra/halb/haproxy.yaml

@@ -3,7 +3,7 @@ hiera_include:
   - keepalived
 
 # keepalived
-profiles::haproxy::dns::vrrp_ipaddr: '198.18.13.250'
+profiles::haproxy::dns::ipaddr: '198.18.13.250'
 profiles::haproxy::dns::vrrp_cnames:
   - sonarr.main.unkin.net
   - radarr.main.unkin.net

hieradata/country/au/region/syd1/infra/halb/haproxy2.yaml

@@ -0,0 +1,254 @@
+---
+profiles::haproxy::dns::ipaddr: "%{hiera('anycast_ip')}"
+profiles::haproxy::dns::vrrp_cnames:
+  - sonarr.main.unkin.net
+  - radarr.main.unkin.net
+  - lidarr.main.unkin.net
+  - readarr.main.unkin.net
+  - prowlarr.main.unkin.net
+  - nzbget.main.unkin.net
+
+profiles::haproxy::mappings:
+  fe_http:
+    ensure: present
+    mappings:
+      - 'au-syd1-pve.main.unkin.net be_ausyd1pve_web'
+      - 'au-syd1-pve-api.main.unkin.net be_ausyd1pve_api'
+      - 'sonarr.main.unkin.net be_sonarr'
+      - 'radarr.main.unkin.net be_radarr'
+      - 'lidarr.main.unkin.net be_lidarr'
+      - 'readarr.main.unkin.net be_readarr'
+      - 'prowlarr.main.unkin.net be_prowlarr'
+      - 'nzbget.main.unkin.net be_nzbget'
+      - 'jellyfin.main.unkin.net be_jellyfin'
+      - 'fafflix.unkin.net be_jellyfin'
+  fe_https:
+    ensure: present
+    mappings:
+      - 'au-syd1-pve.main.unkin.net be_ausyd1pve_web'
+      - 'au-syd1-pve-api.main.unkin.net be_ausyd1pve_api'
+      - 'sonarr.main.unkin.net be_sonarr'
+      - 'radarr.main.unkin.net be_radarr'
+      - 'lidarr.main.unkin.net be_lidarr'
+      - 'readarr.main.unkin.net be_readarr'
+      - 'prowlarr.main.unkin.net be_prowlarr'
+      - 'nzbget.main.unkin.net be_nzbget'
+      - 'jellyfin.main.unkin.net be_jellyfin'
+      - 'fafflix.unkin.net be_jellyfin'
+
+profiles::haproxy::frontends:
+  fe_http:
+    options:
+      use_backend:
+        - "%[req.hdr(host),lower,map(/etc/haproxy/fe_http.map,be_default)]"
+  fe_https:
+    options:
+      acl:
+        - 'acl_ausyd1pve req.hdr(host) -i au-syd1-pve.main.unkin.net'
+        - 'acl_sonarr req.hdr(host) -i sonarr.main.unkin.net'
+        - 'acl_radarr req.hdr(host) -i radarr.main.unkin.net'
+        - 'acl_lidarr req.hdr(host) -i lidarr.main.unkin.net'
+        - 'acl_readarr req.hdr(host) -i readarr.main.unkin.net'
+        - 'acl_prowlarr req.hdr(host) -i prowlarr.main.unkin.net'
+        - 'acl_nzbget req.hdr(host) -i nzbget.main.unkin.net'
+        - 'acl_jellyfin req.hdr(host) -i jellyfin.main.unkin.net'
+        - 'acl_fafflix req.hdr(host) -i fafflix.unkin.net'
+        - 'acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24'
+      use_backend:
+        - "%[req.hdr(host),lower,map(/etc/haproxy/fe_https.map,be_default)]"
+      http-request:
+        - 'deny if { hdr_dom(host) -i au-syd1-pve.main.unkin.net } !acl_internalsubnets'
+      http-response:
+        - 'set-header X-Frame-Options DENY if acl_ausyd1pve'
+        - 'set-header X-Frame-Options DENY if acl_sonarr'
+        - 'set-header X-Frame-Options DENY if acl_radarr'
+        - 'set-header X-Frame-Options DENY if acl_lidarr'
+        - 'set-header X-Frame-Options DENY if acl_readarr'
+        - 'set-header X-Frame-Options DENY if acl_prowlarr'
+        - 'set-header X-Frame-Options DENY if acl_nzbget'
+        - 'set-header X-Frame-Options DENY if acl_jellyfin'
+        - 'set-header X-Frame-Options DENY if acl_fafflix'
+        - 'set-header X-Content-Type-Options nosniff'
+        - 'set-header X-XSS-Protection 1;mode=block'
+
+profiles::haproxy::backends:
+  be_ausyd1pve_web:
+    description: Backend for au-syd1 pve cluster (Web)
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_ausyd1pve_api:
+    description: Backend for au-syd1 pve cluster (API only)
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_sonarr:
+    description: Backend for au-syd1 sonarr
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_radarr:
+    description: Backend for au-syd1 radarr
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_lidarr:
+    description: Backend for au-syd1 lidarr
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_readarr:
+    description: Backend for au-syd1 readarr
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_prowlarr:
+    description: Backend for au-syd1 prowlarr
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_nzbget:
+    description: Backend for au-syd1 nzbget
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /consul/health
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+  be_jellyfin:
+    description: Backend for au-syd1 jellyfin
+    collect_exported: false # handled in custom function
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+        - http-keep-alive
+        - prefer-last-server
+      cookie: SRVNAME insert indirect nocache
+      http-reuse: always
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect: 'scheme https if !{ ssl_fc }'
+
+profiles::haproxy::certlist::enabled: true
+profiles::haproxy::certlist::certificates:
+  - /etc/pki/tls/letsencrypt/au-syd1-pve.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/au-syd1-pve-api.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/sonarr.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/radarr.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/lidarr.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/readarr.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/prowlarr.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/nzbget.main.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/letsencrypt/fafflix.unkin.net/fullchain_combined.pem
+  - /etc/pki/tls/vault/certificate.pem
+
+# additional altnames
+profiles::pki::vault::alt_names:
+  - au-syd1-pve.main.unkin.net
+  - au-syd1-pve-api.main.unkin.net
+  - jellyfin.main.unkin.net
+
+# additional cnames
+profiles::haproxy::dns::cnames:
+  - au-syd1-pve.main.unkin.net
+  - au-syd1-pve-api.main.unkin.net
+
+# letsencrypt certificates
+certbot::client::service: haproxy
+certbot::client::domains:
+  - au-syd1-pve.main.unkin.net
+  - au-syd1-pve-api.main.unkin.net
+  - sonarr.main.unkin.net
+  - radarr.main.unkin.net
+  - lidarr.main.unkin.net
+  - readarr.main.unkin.net
+  - prowlarr.main.unkin.net
+  - nzbget.main.unkin.net
+  - fafflix.unkin.net

hieradata/nodes/ausyd1nxvm1015.main.unkin.net.yaml

@@ -6,7 +6,7 @@ networking::routes:
   default:
     gateway: 198.18.13.254
 
-profiles::haproxy::dns::vrrp_master: true
+profiles::haproxy::dns::master: true
 keepalived::vrrp_instance:
   VI_250:
     state: 'MASTER'

hieradata/roles/infra/halb/haproxy2.yaml

@@ -0,0 +1,181 @@
+---
+hiera_include:
+  - frrouting
+  - profiles::haproxy::server
+
+# networking
+anycast_ip: 198.18.19.17
+systemd::manage_networkd: true
+systemd::manage_all_network_files: true
+networking::interfaces:
+  eth0:
+    type: physical
+    forwarding: true
+    dhcp: true
+  anycast0:
+    type: dummy
+    ipaddress: "%{hiera('anycast_ip')}"
+    netmask: 255.255.255.255
+    mtu: 1500
+
+# frrouting
+frrouting::ospfd_router_id: "%{facts.networking.ip}"
+frrouting::ospfd_redistribute:
+  - connected
+frrouting::ospfd_interfaces:
+  eth0:
+    area: 0.0.0.0
+  anycast0:
+    area: 0.0.0.0
+frrouting::daemons:
+  ospfd: true
+
+# additional repos
+profiles::yum::global::repos:
+  frr-extras:
+    name: frr-extras
+    descr: frr-extras repository
+    target: /etc/yum.repos.d/frr-extras.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/extras-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+  frr-stable:
+    name: frr-stable
+    descr: frr-stable repository
+    target: /etc/yum.repos.d/frr-stable.repo
+    baseurl: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os
+    gpgkey: https://packagerepo.service.consul/frr/el9/stable-daily/%{facts.os.architecture}/os/RPM-GPG-KEY-FRR
+    mirrorlist: absent
+
+# haproxy metrics
+consul::services:
+  haproxy-metrics:
+    service_name: 'haproxy-metrics'
+    tags:
+      - 'metrics'
+      - 'haproxy'
+    address: "%{facts.networking.ip}"
+    port: 8405
+    checks:
+      - id: 'haproxy_metrics_https_check'
+        name: 'HAProxy Metrics Check'
+        http: "https://%{facts.networking.fqdn}:8405/metrics"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: haproxy-metrics
+    disposition: write
+
+# haproxy
+profiles::haproxy::peers::enable: true
+profiles::haproxy::resolvers::enable: true
+profiles::haproxy::ls_stats::port: 9090
+profiles::haproxy::ls_stats::user: 'admin'
+profiles::selinux::setenforce::mode: permissive
+
+profiles::haproxy::server::globals:
+  log:
+    - /dev/log local0
+    - /dev/log local1 notice
+  stats:
+    - timeout 30s
+    - socket /var/lib/haproxy/stats
+    - socket /var/lib/haproxy/admin.sock mode 660 level admin
+  ca-base: /etc/ssl/certs
+  crt-base: /etc/ssl/private
+  ssl-default-bind-ciphers: EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+  ssl-default-bind-options: 'ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3'
+  ssl-default-server-ciphers: kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
+  ssl-default-server-options: no-sslv3
+  tune.ssl.default-dh-param: 2048
+
+profiles::haproxy::server::defaults:
+  mode: http
+  option:
+    - httplog
+    - dontlognull
+    - http-server-close
+    - forwardfor except 127.0.0.0/8
+    - redispatch
+  timeout:
+    - http-request 10s
+    - queue 1m
+    - connect 10s
+    - client 5m
+    - server 5m
+    - http-keep-alive 10s
+    - check 10s
+  retries: 3
+  maxconn: 5000
+
+profiles::haproxy::frontends:
+  fe_http:
+    description: 'Global HTTP Frontend'
+    bind:
+      0.0.0.0:80:
+        - transparent
+    mode: 'http'
+    options:
+      acl:
+        - 'acl-letsencrypt path_beg /.well-known/acme-challenge/'
+      use_backend:
+        - 'be_letsencrypt if acl-letsencrypt'
+      http-request:
+        - 'set-header X-Forwarded-Proto https'
+        - 'set-header X-Real-IP %[src]'
+  fe_https:
+    description: 'Global HTTPS Frontend'
+    bind:
+      0.0.0.0:443:
+        - ssl
+        - crt-list /etc/haproxy/certificate.list
+        - ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+        - force-tlsv12
+    mode: 'http'
+    options:
+      acl:
+        - 'acl-letsencrypt path_beg /.well-known/acme-challenge/'
+      use_backend:
+        - 'be_letsencrypt if acl-letsencrypt'
+      http-request:
+        - 'set-header X-Forwarded-Proto https'
+        - 'set-header X-Real-IP %[src]'
+  fe_metrics:
+    description: 'Metrics Frontend'
+    bind:
+      0.0.0.0:8405:
+        - ssl
+        - crt /etc/pki/tls/vault/certificate.pem
+        - ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+        - force-tlsv12
+    mode: 'http'
+    options:
+      http-request:
+        - 'set-header X-Forwarded-Proto https'
+        - 'set-header X-Real-IP %[src]'
+        - 'use-service prometheus-exporter if { path /metrics }'
+
+profiles::haproxy::backends:
+  be_letsencrypt:
+    description: Backend for LetsEncrypt Verifications
+    collect_exported: true
+    options:
+      balance: roundrobin
+  be_default:
+    description: Backend for unmatched HTTP traffic
+    collect_exported: true
+    options:
+      balance: roundrobin
+      option:
+        - httpchk GET /
+        - forwardfor
+      cookie: SRVNAME insert
+      http-request:
+        - set-header X-Forwarded-Port %[dst_port]
+        - add-header X-Forwarded-Proto https if { dst_port 443 }
+
+prometheus::haproxy_exporter::cnf_scrape_uri: unix:/var/lib/haproxy/stats
+prometheus::haproxy_exporter::export_scrape_job: true

site/profiles/manifests/haproxy/dns.pp

@@ -1,7 +1,6 @@
 # profiles::haproxy::dns
 class profiles::haproxy::dns (
-  Stdlib::IP::Address $vrrp_ipaddr,
-  Boolean             $vrrp_master = false,
+  Stdlib::IP::Address $ipaddr,
   Array[Stdlib::Fqdn] $vrrp_cnames = [],
   Array[Stdlib::Fqdn] $cnames = [],
   Integer             $order  = 10,
@@ -28,24 +27,37 @@ class profiles::haproxy::dns (
     }
   }
 
-  # export a/cnames for haproxy applications
-  if $vrrp_master {
-    profiles::dns::record { "${facts['networking']['fqdn']}_vrrp_${location_environment}-halb-vrrp":
-      value  => $vrrp_ipaddr,
-      type   => 'A',
-      record => "${location_environment}-halb-vrrp",
-      zone   => $::facts['networking']['domain'],
-      order  => $order,
-    }
+  # if it is, find hosts, sort them so they dont cause changes every run
+  $servers_array = sort(query_nodes(
+    "enc_role='${facts['enc_role']}' and
+      country='${facts['country']}' and
+      region='${facts['region']}' and
+      environment='${facts['environment']}'",
+    'networking.fqdn'
+  ))
 
-    $vrrp_cnames.each |$cname| {
-      profiles::dns::record { "${::facts['networking']['fqdn']}_${cname}_CNAME":
-        value  => "${location_environment}-halb-vrrp",
-        type   => 'CNAME',
-        record => "${cname}.",
+  # give enough time for a few hosts to be provisioned
+  if length($servers_array) >= 3 {
+
+    # if this is the first host in the returned filter, export a/cnames for haproxy applications
+    if $servers_array[0] == $trusted['certname'] {
+      profiles::dns::record { "${facts['networking']['fqdn']}_vrrp_${location_environment}-halb-vrrp":
+        value  => $ipaddr,
+        type   => 'A',
+        record => "${location_environment}-halb-vrrp",
         zone   => $::facts['networking']['domain'],
         order  => $order,
       }
+
+      $vrrp_cnames.each |$cname| {
+        profiles::dns::record { "${::facts['networking']['fqdn']}_${cname}_CNAME":
+          value  => "${location_environment}-halb-vrrp",
+          type   => 'CNAME',
+          record => "${cname}.",
+          zone   => $::facts['networking']['domain'],
+          order  => $order,
+        }
+      }
     }
   }
 }

site/profiles/manifests/haproxy/peers.pp

@@ -0,0 +1,24 @@
+# profiles::haproxy::peers
+class profiles::haproxy::peers (
+  Boolean $enable = false,
+){
+
+  if $enable {
+
+    $peer_tag = "${facts['country']}-${facts['region']}-${facts['environment']}"
+
+    @@haproxy::peer { "${peer_tag}_${facts['networking']['fqdn']}":
+      peers_name => $facts['networking']['fqdn'],
+      port       => 10000,
+      tag        => $peer_tag,
+    }
+
+    # collect exported resources
+    Haproxy::Peer <<| tag == $peer_tag |>>
+
+    haproxy::peers { $peer_tag:
+      collect_exported => true
+    }
+
+  }
+}

site/profiles/manifests/haproxy/resolvers.pp

@@ -0,0 +1,20 @@
+# profiles::haproxy::resolvers
+class profiles::haproxy::resolvers (
+  Boolean $enable = false,
+) {
+
+  haproxy::resolver { 'internal':
+    nameservers           => {
+      'dns1' => '198.18.19.16:53',
+    },
+    hold                  => {
+      'nx'    => '30s',
+      'valid' => '10s'
+    },
+    resolve_retries       => 3,
+    timeout               => {
+      'retry' => '1s'
+    },
+    accepted_payload_size => 512,
+  }
+}

site/profiles/manifests/haproxy/server.pp

@@ -36,16 +36,21 @@ class profiles::haproxy::server (
   $merged_default_options = merge($default_options, $defaults)
 
   # wait until enc_role matches haproxy enc_role
-  if $facts['enc_role'] == 'roles::infra::halb::haproxy' {
+  if $facts['enc_role'] in [
+    'roles::infra::halb::haproxy',
+    'roles::infra::halb::haproxy2'
+  ] {
 
     # manage selinux
-    include profiles::haproxy::selinux
+    if $facts['virtual'] != 'lxc' {
+      include profiles::haproxy::selinux
+    }
 
     # create the haproxy service/instance
     class { 'haproxy':
       global_options   => $merged_global_options,
       defaults_options => $merged_default_options,
-      require          => Class['profiles::haproxy::selinux']
+      #require          => Class['profiles::haproxy::selinux']
     }
 
     include certbot::client              # download certbot certs
@@ -53,9 +58,11 @@ class profiles::haproxy::server (
     include profiles::haproxy::mappings  # manage the domain to backend mappings
     include profiles::haproxy::ls_stats  # default status listener
     include profiles::haproxy::dns       # manage dns for haproxy
+    include profiles::haproxy::resolvers # manage resolvers
     include profiles::haproxy::frontends # create frontends
     include profiles::haproxy::backends  # create backends
     include profiles::haproxy::listeners # create listeners
+    include profiles::haproxy::peers     # create peers
     include prometheus::haproxy_exporter # generate metrics
 
     Class['profiles::haproxy::certlist']

site/roles/manifests/infra/halb/haproxy2.pp

@@ -0,0 +1,10 @@
+# a role to deploy an anycast haproxy2 node
+class roles::infra::halb::haproxy2 {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}