Merge branch 'develop' into neoloc/grafana

This commit is contained in:
2024-06-16 00:39:45 +10:00
84 changed files with 983 additions and 125 deletions
+5 -1
View File
@@ -32,6 +32,8 @@ class profiles::base (
include profiles::ntp::client
include profiles::dns::base
include profiles::pki::vault
include profiles::ssh::sign
include profiles::ssh::knownhosts
include profiles::cloudinit::init
include profiles::metrics::default
include profiles::helpers::node_lookup
@@ -56,7 +58,9 @@ class profiles::base (
}
# include classes from hiera
lookup('hiera_classes', Array[String], 'unique').include
$hiera_include = lookup('hiera_include', Array[String], 'unique', [])
$hiera_exclude = lookup('hiera_exclude', Array[String], 'unique', [])
($hiera_include - $hiera_exclude).include
# specifc ordering constraints
Class['profiles::pki::vaultca']
@@ -14,4 +14,11 @@ class profiles::cobbler::service inherits profiles::cobbler::params {
enable => true,
require => File['/etc/httpd/conf.d/ssl.conf'],
}
# ensure tftp is running
service {'tftp':
ensure => 'running',
enable => true,
require => Package['cobbler'],
}
}
+9 -8
View File
@@ -36,14 +36,15 @@ class profiles::consul::client (
# deploy the consul agent
class { 'consul':
config_hash => {
'data_dir' => $data_dir,
'datacenter' => $consul_cluster,
'log_level' => 'INFO',
'node_name' => $facts['networking']['fqdn'],
'retry_join' => $servers_array,
'bind_addr' => $::facts['networking']['ip'],
'advertise_addr' => $::facts['networking']['ip'],
'acl' => {
'data_dir' => $data_dir,
'datacenter' => $consul_cluster,
'log_level' => 'INFO',
'node_name' => $facts['networking']['fqdn'],
'retry_join' => $servers_array,
'bind_addr' => $::facts['networking']['ip'],
'advertise_addr' => $::facts['networking']['ip'],
'enable_script_checks' => true,
'acl' => {
tokens => {
default => fqdn_uuid("${facts['networking']['fqdn']}-${secret_id_salt}")
}
-1
View File
@@ -34,7 +34,6 @@ class profiles::defaults {
ensure => 'present',
enabled => 1,
gpgcheck => 1,
mirrorlist => 'absent',
require => Class['profiles::pki::vaultca'],
notify => Exec['dnf_makecache'],
}
+15 -42
View File
@@ -1,7 +1,13 @@
# profiles::gitea::init
class profiles::gitea::init (
String $mysql_pass = '',
String $lfs_jwt_secret = '',
Hash $root = {},
Hash $server = {},
Hash $database = {},
Hash $repository = {},
Hash $session = {},
Hash $lfs = {},
Hash $ui = {},
) {
include profiles::nginx::simpleproxy
@@ -10,46 +16,13 @@ class profiles::gitea::init (
ensure => '1.22.0',
checksum => 'a31086f073cb9592d28611394b2de3655db515d961e4fdcf5b549cb40753ef3d',
custom_configuration => {
'' => {
'APP_NAME' => 'Gitea',
'RUN_USER' => 'git',
'RUN_MODE' => 'prod',
},
'repository' => {
'ROOT' => '/data/gitea/repos',
'FORCE_PRIVATE' => false,
'MAX_CREATION_LIMIT' => -1,
'DISABLE_HTTP_GIT' => false,
'DEFAULT_BRANCH' => 'main',
'DEFAULT_PRIVATE' => 'last',
},
'ui' => {
'SHOW_USER_EMAIL' => false,
},
'server' => {
'PROTOCOL' => 'http',
'DOMAIN' => 'git.query.consul',
'ROOT_URL' => 'https://git.query.consul',
'HTTP_ADDR' => '0.0.0.0',
'HTTP_PORT' => 3000,
'START_SSH_SERVER' => false,
'SSH_DOMAIN' => 'git.query.consul',
'SSH_PORT' => 2222,
'SSH_LISTEN_HOST' => '0.0.0.0',
'OFFLINE_MODE' => true,
'APP_DATA_PATH' => '/var/lib/gitea/data',
'SSH_LISTEN_PORT' => 22,
},
'database' => {
'DB_TYPE' => 'mysql',
'HOST' => 'mariadb-prod.service.au-syd1.consul:3306',
'NAME' => 'gitea',
'USER' => 'gitea',
'PASSWD' => Sensitive($mysql_pass),
'SSL_MODE' => 'disable',
'PATH' => '/var/lib/gitea/data/gitea.db',
'LOG_SQL' => false,
},
}
'' => $root,
server => $server,
database => $database,
repository => $repository,
session => $session,
lfs => $lfs,
ui => $ui,
},
}
}
@@ -0,0 +1,77 @@
# profiles::helpers::sshsignhost
#
# wrapper class for python, pip and venv
class profiles::helpers::sshsignhost (
String $script_name = 'sshsignhost',
Stdlib::AbsolutePath $base_path = "/opt/${script_name}",
Stdlib::AbsolutePath $venv_path = "${base_path}/venv",
Stdlib::AbsolutePath $config_path = "${base_path}/config.yaml",
Hash $vault_config = {},
String $owner = 'root',
String $group = 'root',
Boolean $systempkgs = false,
String $version = 'system',
Array[String[1]] $packages = ['requests', 'pyyaml'],
){
if $::facts['python3_version'] {
$python_version = $version ? {
'system' => $::facts['python3_version'],
default => $version,
}
# ensure the base_path exists
file { $base_path:
ensure => directory,
mode => '0755',
owner => $owner,
group => $group,
}
# create a venv
python::pyvenv { $venv_path :
ensure => present,
version => $python_version,
systempkgs => $systempkgs,
venv_dir => $venv_path,
owner => $owner,
group => $group,
require => File[$base_path],
}
# install the required pip packages
$packages.each |String $package| {
python::pip { "${venv_path}_${package}":
ensure => present,
pkgname => $package,
virtualenv => $venv_path,
}
}
# create the script from a template
file { "${base_path}/${script_name}":
ensure => file,
mode => '0755',
content => template("profiles/helpers/${script_name}.erb"),
require => Python::Pyvenv[$venv_path],
}
# create the config from a template
file { $config_path:
ensure => file,
mode => '0660',
owner => 'puppet',
group => 'root',
content => Sensitive(template("profiles/helpers/${script_name}_config.yaml.erb")),
require => Python::Pyvenv[$venv_path],
}
# create symbolic link in $PATH
file { "/usr/local/bin/${script_name}":
ensure => 'link',
target => "${base_path}/${script_name}",
require => File["${base_path}/${script_name}"],
}
}
}
+8
View File
@@ -35,5 +35,13 @@ class profiles::ntp::server (
queryhosts => $allowquery,
}
}
file {'/usr/local/bin/check_ntp.sh':
ensure => 'file',
owner => 'root',
group => 'root',
mode => '0755',
content => template('profiles/ntp/check_ntp.sh.erb'),
}
}
}
+1
View File
@@ -12,6 +12,7 @@ class profiles::puppet::client (
Integer $runtimeout = 3600,
Boolean $show_diff = true,
Boolean $usecacheonfailure = false,
Integer $facts_soft_limit = 4096,
) {
# dont manage puppet.conf if this is a puppetmaster
+4 -25
View File
@@ -10,32 +10,12 @@ class profiles::puppet::enc (
Boolean $force = false,
) {
vcsrepo { '/opt/puppetlabs/enc':
ensure => latest,
provider => git,
source => $repo,
revision => $release,
force => $force,
require => Package['git'],
}
file { '/opt/puppetlabs/bin/enc':
ensure => link,
target => '/opt/puppetlabs/enc/enc.py',
require => Vcsrepo['/opt/puppetlabs/enc'],
ensure => absent,
}
file { '/opt/puppetlabs/bin/puppet-enc':
ensure => file,
owner => 'root',
group => 'root',
mode => '0755',
content => "#!/bin/bash\n(
cd /opt/puppetlabs/enc/
git reset --hard master
git clean -fd
git pull\n)",
require => Package['git'],
ensure => absent,
}
$_timer = @(EOT)
@@ -63,8 +43,7 @@ class profiles::puppet::enc (
systemd::timer { 'puppet-enc.timer':
timer_content => $_timer,
service_content => $_service,
active => true,
enable => true,
require => File['/opt/puppetlabs/bin/puppet-enc'],
active => false,
enable => false,
}
}
+25 -1
View File
@@ -21,16 +21,40 @@ class profiles::puppet::puppetca (
# manage the crl file
if $is_puppetca {
# export the puppet crl.pem
@@file { '/etc/puppetlabs/puppet/ssl/crl.pem':
@@file { '/etc/puppetlabs/puppet/ssl/crl.pem.latest':
ensure => file,
content => file('/etc/puppetlabs/puppet/ssl/crl.pem'),
tag => 'crl_pem_export',
}
systemd::manage_dropin { 'copy_crl.conf':
ensure => absent,
unit => 'puppetserver.service',
}
}else{
# import the puppet crl.pem
File <<| tag == 'crl_pem_export' |>> {
require => Service['puppetserver'],
}
# copy latest to active location
file { '/etc/puppetlabs/puppet/ssl/crl.pem':
ensure => file,
owner => 'puppet',
group => 'puppet',
source => '/etc/puppetlabs/puppet/ssl/crl.pem.latest',
require => File['/etc/puppetlabs/puppet/ssl/crl.pem.latest'],
}
# copy the latest crl when restarting
systemd::manage_dropin { 'copy_crl.conf':
ensure => present,
unit => 'puppetserver.service',
service_entry => {
'ExecStartPost' => [
'/usr/bin/sleep 2',
'/bin/cp /etc/puppetlabs/puppet/ssl/crl.pem.latest /etc/puppetlabs/puppet/ssl/crl.pem',
],
},
require => File['/etc/puppetlabs/puppet/ssl/crl.pem'],
}
}
# register the PuppetCA service with consul
+21 -3
View File
@@ -2,6 +2,7 @@
class profiles::puppet::puppetdb_sql (
String $puppetdb_host = lookup('puppetdbsql'),
String $listen_address = $facts['networking']['ip'],
String $consul_test_db_pass = '',
) {
# disable the postgresql dnf module for el8+
@@ -17,9 +18,11 @@ class profiles::puppet::puppetdb_sql (
# Install and configure PostgreSQL for PuppetDB
class { 'puppetdb::database::postgresql':
listen_addresses => $listen_address,
postgres_version => '15',
puppetdb_server => $puppetdb_host,
listen_addresses => $listen_address,
postgres_version => '15',
puppetdb_server => $puppetdb_host,
manage_package_repo => false,
require => [ Yumrepo['postgresql-15'],Yumrepo['postgresql-common'] ],
}
contain ::puppetdb::database::postgresql
@@ -32,4 +35,19 @@ class profiles::puppet::puppetdb_sql (
value => $value,
}
}
# create consul database + user to test the host is responsive
postgresql::server::db { 'consul_test_db':
user => 'consul_test_user',
password => postgresql::postgresql_password('consul_test_user', Sensitive($consul_test_db_pass) ),
}
file { '/usr/local/bin/check_consul_postgresql':
ensure => 'file',
owner => 'root',
group => 'root',
mode => '0755',
content => template('profiles/puppetdb/check_consul_postgresql.erb'),
before => Class['profiles::consul::client'],
}
}
@@ -15,6 +15,7 @@ class profiles::puppet::puppetmaster (
include profiles::puppet::autosign
include profiles::puppet::gems
include profiles::helpers::certmanager
include profiles::helpers::sshsignhost
include profiles::puppet::server
include profiles::puppet::puppetca
include profiles::puppet::eyaml
+12
View File
@@ -28,6 +28,7 @@ class profiles::puppet::server (
Integer $runinterval = 1800,
Integer $runtimeout = 3600,
Boolean $show_diff = true,
Integer $facts_soft_limit = 4096,
) {
file { '/etc/puppetlabs/puppet/puppet.conf':
@@ -59,6 +60,7 @@ class profiles::puppet::server (
'storeconfigs_backend' => $storeconfigs_backend,
'reports' => $reports,
'usecacheonfailure' => $usecacheonfailure,
'facts_soft_limit' => $facts_soft_limit,
}),
notify => Service['puppetserver'],
}
@@ -69,4 +71,14 @@ class profiles::puppet::server (
hasstatus => true,
hasrestart => true,
}
# generate puppet types when restarting
systemd::manage_dropin { 'generate_types.conf':
ensure => present,
unit => 'puppetserver.service',
service_entry => {
'ExecStartPost' => [
"/opt/puppetlabs/bin/puppet generate types --environmentpath ${codedir}/environments",
],
},
}
}
+12
View File
@@ -0,0 +1,12 @@
# manage known hosts
class profiles::ssh::knownhosts (
Array $lines = [],
) {
file {'/etc/ssh/ssh_known_hosts':
ensure => 'file',
owner => 'root',
group => 'root',
mode => '0644',
content => template('profiles/ssh/ssh_known_hosts.erb'),
}
}
+84
View File
@@ -0,0 +1,84 @@
# profiles::ssh::sign
class profiles::ssh::sign (
Optional[Array[Stdlib::Host]] $principals = [],
){
# validate and prepare additional alt_names, if any
$default_principals = [
$::facts['networking']['hostname'],
$::facts['networking']['fqdn'],
$::facts['networking']['ip'],
]
$effective_principals = $principals ? {
[] => $default_principals,
default => concat($default_principals, $principals),
}
# path for the principals file
$principals_file = '/etc/ssh/host_principals'
# alt_names_file contents
$principals_file_content = $effective_principals
# manage the alt names file
file { $principals_file:
ensure => file,
owner => 'root',
group => 'root',
mode => '0644',
content => join($principals_file_content, "\n"),
}
# compare the sorted arrays of principals from disk (fact) vs what is intended (this run)
$principals_match = sort($::facts['sshd_host_principals']) == sort($principals_file_content)
# only renew signed certificate if doesnt exist or the principals have changed
if ! $::facts['sshd_host_cert_exists'] or ! $principals_match {
$common_name = $::facts['networking']['fqdn']
$valid_hours = '87600h'
# prepare alt_names and ip_sans arguments conditionally
$principals_string = $effective_principals.empty() ? {
true => '',
default => join($effective_principals, ','),
}
# sshsignhost arguments
$cmd = '/usr/local/bin/sshsignhost'
$principals_arg = '--valid_principals'
$ttl_arg = '--ttl'
$public_key_arg = '--public_key'
# call the script with generate(), capturing json output
$json_output = generate(
$cmd,
$principals_arg,
$principals_string,
$ttl_arg,
$valid_hours,
$public_key_arg,
"${facts['ssh']['rsa']['type']} ${facts['ssh']['rsa']['key']}",
'--json'
)
$signed_data = parsejson($json_output)
# manage the signed hostkey file
file { '/etc/ssh/ssh_host_rsa_key-cert.pem':
ensure => file,
content => $signed_data['signed_key'],
owner => 'root',
group => 'root',
mode => '0644',
}
}else{
# manage the signed hostkey file
file { '/etc/ssh/ssh_host_rsa_key-cert.pem':
ensure => file,
owner => 'root',
group => 'root',
mode => '0644',
}
}
}
+11 -3
View File
@@ -16,6 +16,13 @@ class profiles::yum::global (
purge => $purge,
}
#exec {'purge_almalinux_default_repos':
# command => 'rm -f /etc/yum.repos.d/almalinux*.repo',
# path => ['/bin', '/usr/bin'],
# onlyif => 'find /etc/yum.repos.d/ -type f -name *almalinux* | grep .',
# before => Resources['yumrepo'],
#}
# download all gpg keys if a repo defines it
$repos.each |$name, $repo| {
if $repo['gpgkey'] {
@@ -29,11 +36,12 @@ class profiles::yum::global (
before => Yumrepo[$name],
}
}
# create the repo
yumrepo { $name:
* => $repo,
}
}
# create repos
create_resources('yumrepo', $repos)
# makecache if changes made to repos
exec {'dnf_makecache':
command => 'dnf makecache -q',
@@ -0,0 +1,83 @@
#!<%= @venv_path %>/bin/python
import argparse
import requests
import json
import yaml
# remove this after certs are generated everywhere
requests.packages.urllib3.disable_warnings()
def load_config(config_path):
with open(config_path, 'r') as file:
config = yaml.safe_load(file)
return config['vault']
def authenticate_approle(vault_config):
url = f"{vault_config['addr']}/v1/auth/{vault_config['approle_path']}/login"
payload = {
"role_id": vault_config['role_id'],
}
response = requests.post(url, json=payload, verify=False)
if response.status_code == 200:
auth_response = response.json()
return auth_response['auth']['client_token']
else:
print(f"Error authenticating with AppRole: {response.text}")
return None
def sign_ssh_certificate(vault_config, public_key, valid_principals, ttl):
# Authenticate using AppRole and get a token
client_token = authenticate_approle(vault_config)
if not client_token:
print("Failed to authenticate with Vault using AppRole.")
return None
# Prepare the SSH certificate signing request
url = f"{vault_config['addr']}/v1/{vault_config['mount_point']}/sign/{vault_config['role_name']}"
headers = {'X-Vault-Token': client_token}
payload = {
"cert_type": "host",
"public_key": public_key,
"valid_principals": valid_principals,
"ttl": ttl
}
# Request the SSH certificate signing
response = requests.post(url, headers=headers, json=payload, verify=False)
if response.status_code == 200:
return response.json()
else:
print(f"Error requesting certificate: {response.text}")
return None
def main(config_file):
config = load_config(config_file)
parser = argparse.ArgumentParser(description='Sign SSH host certificate using Vault.')
parser.add_argument('--public_key', required=True, help='SSH public key as a string')
parser.add_argument('--valid_principals', required=True, help='Comma-separated list of valid principals')
parser.add_argument('--ttl', default='87600h', help='Time-to-live for the certificate (default: 87600h)')
parser.add_argument('--json', action='store_true', help='Output the resulting certificate as JSON')
args = parser.parse_args()
# Load configuration
config = load_config(config_file)
# Sign SSH certificate
response = sign_ssh_certificate(config, args.public_key, args.valid_principals, args.ttl)
if response and 'data' in response and 'signed_key' in response['data']:
if args.json:
output = {
'signed_key': response['data']['signed_key'],
}
print(json.dumps(output))
else:
print(response['data']['signed_key'])
else:
print("Error: The response does not contain the expected data.")
exit(1)
if __name__ == "__main__":
config_file = '<%= @config_path %>'
main(config_file)
@@ -0,0 +1,7 @@
vault:
addr: '<%= @vault_config['addr'] %>'
role_id: '<%= @vault_config['role_id'] %>'
approle_path: '<%= @vault_config['approle_path'] %>'
mount_point: '<%= @vault_config['mount_point'] %>'
role_name: '<%= @vault_config['role_name'] %>'
output_path: '<%= @vault_config['output_path'] %>'
@@ -0,0 +1,8 @@
#!/usr/bin/bash
# Check if ntpd or chronyd is running
if pgrep ntpd > /dev/null || pgrep chronyd > /dev/null; then
exit 0
else
exit 2
fi
@@ -11,3 +11,4 @@ runinterval = <%= @runinterval %>
runtimeout = <%= @runtimeout %>
show_diff = <%= @show_diff %>
usecacheonfailure = <%= @usecacheonfailure %>
number_of_facts_soft_limit = <%= @facts_soft_limit %>
@@ -17,6 +17,7 @@ report_server = <%= $report_server %>
runinterval = <%= $runinterval %>
runtimeout = <%= $runtimeout %>
show_diff = <%= $show_diff %>
number_of_facts_soft_limit = <%= $facts_soft_limit %>
[master]
node_terminus = <%= $node_terminus %>
@@ -0,0 +1,2 @@
#!/usr/bin/bash
PGPASSWORD=<%= @consul_test_db_pass %> /usr/bin/psql -U consul_test_user -d consul_test_db -h <%= @facts['networking']['ip'] %> -p 5432 -c "SELECT 1"
@@ -0,0 +1,4 @@
# this file is managed by puppet
<% @lines.each do |line| -%>
<%= line %>
<% end -%>
+3 -1
View File
@@ -6,6 +6,8 @@ class roles::infra::puppetdb::sql {
}else{
include profiles::defaults
include profiles::base
include profiles::puppet::puppetdb_sql
if $facts['enc_role'] == 'roles::infra::puppetdb::sql' {
include profiles::puppet::puppetdb_sql
}
}
}