From aadd0275ac7e840b38de4e3a9ba5c236111176ac Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Thu, 8 Aug 2024 19:28:50 +1000 Subject: [PATCH 1/3] feat: add puppet-redis module --- Puppetfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Puppetfile b/Puppetfile index 9677851..ead9ecd 100644 --- a/Puppetfile +++ b/Puppetfile @@ -40,6 +40,7 @@ mod 'puppet-kmod', '4.0.1' mod 'puppet-filemapper', '4.0.0' mod 'puppet-letsencrypt', '11.0.0' mod 'puppet-rundeck', '9.1.0' +mod 'puppet-redis', '11.0.0' # other mod 'ghoneycutt-puppet', '3.3.0' -- 2.47.3 From b7fc6a19931e4689d2849c0214a3e46c79bffbb0 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Fri, 9 Aug 2024 23:22:37 +1000 Subject: [PATCH 2/3] feat: create redisha module - manage redis/sentinel clusters - ensure ulimit_managed is false - dynamically find servers in role to identify master - add redisadm and sentineladm commands - add script to check if the current host in the master --- modules/redisha/manifests/init.pp | 14 +++++ modules/redisha/manifests/params.pp | 25 ++++++++ modules/redisha/manifests/redis.pp | 59 +++++++++++++++++++ modules/redisha/manifests/sentinel.pp | 49 +++++++++++++++ modules/redisha/manifests/tools.pp | 35 +++++++++++ .../redisha/templates/check_redis_master.erb | 2 + modules/redisha/templates/redisadm.erb | 9 +++ modules/redisha/templates/sentineladm.erb | 9 +++ 8 files changed, 202 insertions(+) create mode 100644 modules/redisha/manifests/init.pp create mode 100644 modules/redisha/manifests/params.pp create mode 100644 modules/redisha/manifests/redis.pp create mode 100644 modules/redisha/manifests/sentinel.pp create mode 100644 modules/redisha/manifests/tools.pp create mode 100644 modules/redisha/templates/check_redis_master.erb create mode 100644 modules/redisha/templates/redisadm.erb create mode 100644 modules/redisha/templates/sentineladm.erb diff --git a/modules/redisha/manifests/init.pp b/modules/redisha/manifests/init.pp new file mode 100644 index 0000000..c1df17a --- /dev/null +++ b/modules/redisha/manifests/init.pp @@ -0,0 +1,14 @@ +# manage RedisHA +class redisha ( + Boolean $manage_repo = $redisha::params::manage_repo, + Boolean $redisha_members_lookup = $redisha::params::redisha_members_lookup, + Optional[String] $redisha_members_role = $redisha::params::redisha_members_role, + Array $redisha_servers = $redisha::params::redisha_servers, +) inherits redisha::params { + + include redisha::redis + include redisha::sentinel + include redisha::tools + + Class['redisha::redis'] -> Class['redisha::sentinel'] -> Class['redisha::tools'] +} diff --git a/modules/redisha/manifests/params.pp b/modules/redisha/manifests/params.pp new file mode 100644 index 0000000..bedbf19 --- /dev/null +++ b/modules/redisha/manifests/params.pp @@ -0,0 +1,25 @@ +class redisha::params ( + Boolean $redisha_members_lookup = false, + Optional[String] $redisha_members_role = undef, + Array $redisha_servers = [], + + # both + Stdlib::Host $redis_host = $facts['networking']['ip'], + Stdlib::Port $redis_port = 6379, + Optional[String] $requirepass = undef, + + # redis + Optional[String] $dnf_module_stream = '6', + Integer[1] $databases = 16, + Optional[Variant[String, Sensitive[String], Deferred]] $masterauth = $redisha::params::requirepass, + + # sentinel + String[1] $master_name = 'mymaster', + Optional[Variant[String, Sensitive[String]]] $auth_pass = $redisha::params::requirepass, + Integer[1] $quorum = 2, + Enum['yes', 'no'] $sentinel_resolve_hostnames = 'yes', + Enum['yes', 'no'] $sentinel_announce_hostnames = 'yes', + Stdlib::Host $sentinel_announce_ip = $facts['networking']['ip'], + Array[Stdlib::IP::Address] $sentinel_bind = [$facts['networking']['ip']], + Stdlib::Port $sentinel_port = 26379, +){} diff --git a/modules/redisha/manifests/redis.pp b/modules/redisha/manifests/redis.pp new file mode 100644 index 0000000..cf375e4 --- /dev/null +++ b/modules/redisha/manifests/redis.pp @@ -0,0 +1,59 @@ +class redisha::redis ( + Boolean $manage_repo = $redisha::manage_repo, + Boolean $redisha_members_lookup = $redisha::redisha_members_lookup, + Optional[String] $redisha_members_role = $redisha::redisha_members_role, + Array $redisha_servers = $redisha::redisha_servers, + Stdlib::Host $redis_host = $redisha::params::redis_host, + Stdlib::Port $redis_port = $redisha::params::redis_port, + Optional[String] $requirepass = $redisha::params::requirepass, + Optional[String] $dnf_module_stream = $redisha::params::dnf_module_stream, + Integer[1] $databases = $redisha::params::databases, + Optional[Variant[String, Sensitive[String], Deferred]] $masterauth = $redisha::params::masterauth, +) inherits redisha::params { + + # if lookup is enabled + if $redisha_members_lookup { + + # check that the role is also set + unless !($redisha_members_role == undef) { + fail("redisha_members_role must be provided for ${title} when redisha_members_lookup is True") + } + + # if it is, find hosts, sort them so they dont cause changes every run + $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn')) + + # else use provided array from params + }else{ + $servers_array = $redisha_servers + } + + + if length($servers_array) >= 3 { + + # check if this is the master_node + if $servers_array[0] == $::facts['networking']['fqdn'] { + class { 'redis': + bind => $redis_host, + port => $redis_port, + databases => $databases, + requirepass => $requirepass, + masterauth => $masterauth, + dnf_module_stream => $dnf_module_stream, + ulimit_managed => false, + } + }else{ + class { 'redis': + bind => $redis_host, + port => $redis_port, + databases => $databases, + requirepass => $requirepass, + masterauth => $masterauth, + dnf_module_stream => $dnf_module_stream, + ulimit_managed => false, + replicaof => "${servers_array[0]} ${redis_port}", + } + } + + } + +} diff --git a/modules/redisha/manifests/sentinel.pp b/modules/redisha/manifests/sentinel.pp new file mode 100644 index 0000000..9ef4d42 --- /dev/null +++ b/modules/redisha/manifests/sentinel.pp @@ -0,0 +1,49 @@ +class redisha::sentinel ( + Boolean $redisha_members_lookup = $redisha::redisha_members_lookup, + Optional[String] $redisha_members_role = $redisha::redisha_members_role, + Array $redisha_servers = $redisha::redisha_servers, + Stdlib::Port $redis_port = $redisha::params::redis_port, + Optional[String] $requirepass = $redisha::params::requirepass, + String[1] $master_name = $redisha::params::master_name, + Optional[Variant[String, Sensitive[String]]] $auth_pass = $redisha::params::auth_pass, + Integer[1] $quorum = $redisha::params::quorum, + Enum['yes', 'no'] $sentinel_resolve_hostnames = $redisha::params::sentinel_resolve_hostnames, + Enum['yes', 'no'] $sentinel_announce_hostnames = $redisha::params::sentinel_announce_hostnames, + Stdlib::Host $sentinel_announce_ip = $redisha::params::sentinel_announce_ip, + Array[Stdlib::IP::Address] $sentinel_bind = $redisha::params::sentinel_bind, + Stdlib::Port $sentinel_port = $redisha::params::sentinel_port, +) inherits redisha::params { + + # if lookup is enabled + if $redisha_members_lookup { + + # check that the role is also set + unless !($redisha_members_role == undef) { + fail("redisha_members_role must be provided for ${title} when redisha_members_lookup is True") + } + + # if it is, find hosts, sort them so they dont cause changes every run + $servers_array = sort(query_nodes("enc_role='${redisha_members_role}' and region='${facts['region']}'", 'networking.fqdn')) + + # else use provided array from params + }else{ + $servers_array = $redisha_servers + } + + if length($servers_array) >= 3 { + + class { 'redis::sentinel': + master_name => $master_name, + redis_host => $servers_array[0], + redis_port => $redis_port, + requirepass => $requirepass, + auth_pass => $auth_pass, + quorum => $quorum, + sentinel_resolve_hostnames => $sentinel_resolve_hostnames, + sentinel_announce_ip => $sentinel_announce_ip, + sentinel_announce_hostnames => $sentinel_announce_hostnames, + sentinel_port => $sentinel_port, + sentinel_bind => $sentinel_bind, + } + } +} diff --git a/modules/redisha/manifests/tools.pp b/modules/redisha/manifests/tools.pp new file mode 100644 index 0000000..7b115e9 --- /dev/null +++ b/modules/redisha/manifests/tools.pp @@ -0,0 +1,35 @@ +class redisha::tools ( + Stdlib::Host $redis_host = $redisha::params::redis_host, + Stdlib::Port $redis_port = $redisha::params::redis_port, + Stdlib::Port $sentinel_port = $redisha::params::sentinel_port, + Optional[String] $requirepass = $redisha::params::requirepass, +) inherits redisha::params { + + # add command to automate redis-cli commands against redis + file {'/usr/local/sbin/redisadm': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0700', + content => template('redisha/redisadm.erb'), + } + + # add command to automate redis-cli commands against sentinel + file {'/usr/local/sbin/sentineladm': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0700', + content => template('redisha/sentineladm.erb'), + } + + # add command to check if current host is the redis master + file {'/usr/local/bin/check_redis_master': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0755', + content => template('redisha/check_redis_master.erb'), + } + +} diff --git a/modules/redisha/templates/check_redis_master.erb b/modules/redisha/templates/check_redis_master.erb new file mode 100644 index 0000000..8e69a76 --- /dev/null +++ b/modules/redisha/templates/check_redis_master.erb @@ -0,0 +1,2 @@ +#!/usr/bin/bash +sudo /usr/local/sbin/sentineladm info | grep -q <%= @facts['networking']['fqdn'] %> diff --git a/modules/redisha/templates/redisadm.erb b/modules/redisha/templates/redisadm.erb new file mode 100644 index 0000000..7d976a3 --- /dev/null +++ b/modules/redisha/templates/redisadm.erb @@ -0,0 +1,9 @@ +#!/usr/bin/bash +REDIS_HOST=<%= @redis_host %> +REDIS_PORT=<%= @redis_port %> + +if [ $# -gt 0 ]; then + REDISCLI_AUTH=<%= @requirepass %> redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" "$@" +else + REDISCLI_AUTH=<%= @requirepass %> redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" +fi diff --git a/modules/redisha/templates/sentineladm.erb b/modules/redisha/templates/sentineladm.erb new file mode 100644 index 0000000..bd98fd0 --- /dev/null +++ b/modules/redisha/templates/sentineladm.erb @@ -0,0 +1,9 @@ +#!/usr/bin/bash +REDIS_HOST=<%= @redis_host %> +SENTINEL_PORT=<%= @sentinel_port %> + +if [ $# -gt 0 ]; then + REDISCLI_AUTH=<%= @requirepass %> redis-cli -h "$REDIS_HOST" -p "$SENTINEL_PORT" "$@" +else + REDISCLI_AUTH=<%= @requirepass %> redis-cli -h "$REDIS_HOST" -p "$SENTINEL_PORT" +fi -- 2.47.3 From 5c731fef34ae30b20ea055fab32dbe7b3ca418d5 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sat, 10 Aug 2024 14:25:28 +1000 Subject: [PATCH 3/3] feat: deploy redisha cluster - manage pki and ssh principals - manage redis/sentinel with redisha module - add consul checks to manage redis-replica/redis-master services - manage sudo rules for consul checks --- hieradata/roles/infra/db/redis.eyaml | 2 + hieradata/roles/infra/db/redis.yaml | 67 ++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 hieradata/roles/infra/db/redis.eyaml create mode 100644 hieradata/roles/infra/db/redis.yaml diff --git a/hieradata/roles/infra/db/redis.eyaml b/hieradata/roles/infra/db/redis.eyaml new file mode 100644 index 0000000..13c4ad6 --- /dev/null +++ b/hieradata/roles/infra/db/redis.eyaml @@ -0,0 +1,2 @@ +--- +redisha::masterauth: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAV8znsSGAbPpPUhAcyOIWFltVyAcx3yVcIvC+JFndkkuVBT1813GSURrXIreXSilvJEHlwRC03A9NhjWJSsHBIS12uUb+7ap95oh2JJ7OHmeWSVD1GDDRpTQAgDEOikAnioRNJfQ83jUa11nJrsavt46hSq8vDq+rZ2P8ugiNk59mNX5vgYthCPXcEJd7UmpLZhgxZ8+42l4TKo7QpqKRcIMteJXk1NvyYfYGnGTZhknuyHM3xPGauGjKzamMlTzD9dGnn2K0/Q7I4PUyT24ZEG3kNVyDlVHTYcKnIT5q8qvmJdDQfZwOETF3SrzcqhQ2nqFvmI19sCTsQVmveb/2ITBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC9KML3vzwF4vGZdtiu++jmgDAWPZspCckgoXPkgNRMePov3pXSlKUAGxwmdsuIM75rJMxlSTiil2lzMMBWXmUofys=] diff --git a/hieradata/roles/infra/db/redis.yaml b/hieradata/roles/infra/db/redis.yaml new file mode 100644 index 0000000..5aaedc8 --- /dev/null +++ b/hieradata/roles/infra/db/redis.yaml @@ -0,0 +1,67 @@ +--- +# additional altnames +profiles::pki::vault::alt_names: + - redis.main.unkin.net + - redis.service.consul + - redis.query.consul + - "redis.service.%{facts.country}-%{facts.region}.consul" + +profiles::ssh::sign::principals: + - redis.main.unkin.net + - redis.service.consul + - redis.query.consul + + +hiera_include: + - redisha + +redisha::manage_repo: false +redisha::redisha_members_lookup: true +redisha::redisha_members_role: roles::infra::db::redis +redisha::redis::requirepass: "%{hiera('redisha::masterauth')}" +redisha::redis::masterauth: "%{hiera('redisha::masterauth')}" +redisha::sentinel::master_name: "%{facts.country}-%{facts.region}" +redisha::sentinel::requirepass: "%{hiera('redisha::masterauth')}" +redisha::sentinel::auth_pass: "%{hiera('redisha::masterauth')}" +redisha::tools::requirepass: "%{hiera('redisha::masterauth')}" + +sudo::configs: + consul: + priority: 20 + content: | + consul ALL=(ALL) NOPASSWD: /usr/local/sbin/sentineladm info +consul::services: + redis-replica: + service_name: "redis-replica-%{facts.environment}" + tags: + - 'redis' + - 'redis-replica' + address: "%{facts.networking.ip}" + port: 6379 + checks: + - id: 'redis-replica_tcp_check' + name: 'Redis Replica TCP Check' + tcp: "%{facts.networking.ip}:6379" + interval: '10s' + timeout: '1s' + redis-master: + service_name: "redis-master-%{facts.environment}" + tags: + - 'redis' + - 'redis-master' + address: "%{facts.networking.ip}" + port: 6379 + checks: + - id: 'redis-master_tcp_check' + name: "Redis Master Check" + args: + - '/usr/local/bin/check_redis_master' + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: "redis-replica-%{facts.environment}" + disposition: write + - resource: service + segment: "redis-master-%{facts.environment}" + disposition: write -- 2.47.3