feat: adding reposync wrapper and tooling

- add autosyncer/autopromoter scripts
- add timer and service to initial sync process
- add timer/service for daily/weekly/monthly autopromote
- add define to manage each repo
- add nginx webserver to share repos
- add favion.ico if enabled
- add selinux management, and packages for selinux
- cleanup package management, sorting package groups into package classes

Puppetfile

@@ -22,6 +22,7 @@ mod 'puppet-archive', '7.0.0'
 mod 'puppet-chrony', '2.6.0'
 mod 'puppet-puppetboard', '9.0.0'
 mod 'puppet-nginx', '5.0.0'
+mod 'puppet-selinux', '4.1.0'
 
 # other
 mod 'ghoneycutt-puppet', '3.3.0'

hieradata/common.yaml

@@ -6,7 +6,7 @@ profiles::base::ntp_servers:
 profiles::base::puppet_servers:
   - 'prodinf01n01.main.unkin.net'
 
-profiles::base::packages::common:
+profiles::packages::base:
   - ccze
   - curl
   - dstat
@@ -14,6 +14,7 @@ profiles::base::packages::common:
   - mtr
   - ncdu
   - neovim
+  - rsync
   - screen
   - strace
   - tmux
@@ -56,6 +57,42 @@ puppetdb::master::config::create_puppet_service_resource: false
 profiles::accounts::sysadmin::sshkeys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ8SRLlPiDylBpdWR9LpvPg4fDVD+DZst4yRPFwMMhta4mnB1H9XuvZkptDhXywWQ7QIcqa2WbhCen0OQJCtwn3s7EYtacmF5MxmwBYocPoK2AArGuh6NA9rwTdLrPdzhZ+gwe88PAzRLNzjm0ZBR+mA9saMbPJdqpKp0AWeAM8QofRQAWuCzQg9i0Pn1KDMvVDRHCZof4pVlHSTyHNektq4ifovn0zhKC8jD/cYu95mc5ftBbORexpGiQWwQ3HZw1IBe0ZETB1qPIPwsoJpt3suvMrL6T2//fcIIUE3TcyJKb/yhztja4TZs5jT8370G/vhlT70He0YPxqHub8ZfBv0khlkY93VBWYpNGJwM1fVqlw7XbfBNdOuJivJac8eW317ZdiDnKkBTxapThpPG3et9ib1HoPGKRsd/fICzNz16h2R3tddSdihTFL+bmTCa6Lo+5t5uRuFjQvhSLSgO2/gRAprc3scYOB4pY/lxOFfq3pU2VvSJtRgLNEYMUYKk= ben@unkin.net
 
+profiles::reposync::repos_list:
+  almalinux_8_8_baseos:
+    repository: 'BaseOS'
+    description: 'AlmaLinux 8.8 - BaseOS'
+    osname: 'almalinux'
+    release: '8.8'
+    baseurl: 'http://mirror.aarnet.edu.au/pub/almalinux/8.8/BaseOS/x86_64/os/'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux'
+  almalinux_8_8_appstream:
+    repository: 'AppStream'
+    description: 'AlmaLinux 8.8 - AppStream'
+    osname: 'almalinux'
+    release: '8.8'
+    baseurl: 'http://mirror.aarnet.edu.au/pub/almalinux/8.8/AppStream/x86_64/os/'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux'
+  almalinux_8_8_highavailability:
+    repository: 'HighAvailability'
+    description: 'AlmaLinux 8.8 - HighAvailability'
+    osname: 'almalinux'
+    release: '8.8'
+    baseurl: 'http://mirror.aarnet.edu.au/pub/almalinux/8.8/HighAvailability/x86_64/os/'
+    gpgkey: 'http://mirror.aarnet.edu.au/pub/almalinux/RPM-GPG-KEY-AlmaLinux'
+  epel_8_everything:
+    repository: 'Everything'
+    description: 'EPEL 8 Everything'
+    osname: 'epel'
+    release: '8'
+    baseurl: 'https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/'
+    gpgkey: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8'
+  epel_8_modular:
+    repository: 'Modular'
+    description: 'EPEL 8 Modular'
+    osname: 'epel'
+    release: '8'
+    baseurl: 'https://dl.fedoraproject.org/pub/epel/8/Modular/x86_64/'
+    gpgkey: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8'
 
 profiles::base::hosts::additional_hosts:
   - ip: 198.18.17.3
@@ -78,3 +115,8 @@ profiles::base::hosts::additional_hosts:
     hostname: prodinf01n06.main.unkin.net
     aliases:
       - prodinf01n06
+  - ip: 198.18.17.22
+    hostname: prodinf01n22.main.unkin.net
+    aliases:
+      - prodinf01n22
+      - repo.main.unkin.net

site/profiles/manifests/base.pp

@@ -19,22 +19,18 @@ class profiles::base (
     }
   }
 
-  # include the base packages profile
-  class { 'profiles::base::packages':
-    packages => hiera('profiles::base::packages::common'),
-    ensure   => 'installed',
-  }
-
   # manage puppet clients
   if ! member($puppet_servers, $trusted['certname']) {
     include profiles::puppet::client
   }
 
-  # include admin scripts
+  # include the base profiles
+  include profiles::packages::base
+  include profiles::base::facts
+  include profiles::base::motd
   include profiles::base::scripts
-
-  # include admin scripts
   include profiles::base::hosts
+  include profiles::accounts::sysadmin
 
   # include the python class
   class { 'python':
@@ -49,11 +45,4 @@ class profiles::base (
     secure_path => '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/puppetlabs/bin'
   }
 
-  # default users
-  include profiles::accounts::sysadmin
-
-  # add a motd
-  include profiles::base::facts
-  include profiles::base::motd
-
 }

site/profiles/manifests/base/packages.pp

@@ -1,27 +0,0 @@
-# This class manages the installation of packages for the base profile
-#
-# Parameters:
-# - $packages: An array of package names to be installed (optional)
-#
-# Description:
-# This class installs a list of packages specified in the $packages parameter
-# using the `package` resource from Puppet. Each package in the array is installed
-# with the `ensure => installed` attribute, ensuring that the package is present
-# on the target system. By default, the class retrieves the package list from Hiera
-# using the key 'profiles::base::packages::common'.
-#
-# Example usage:
-# class { 'profiles::base::packages':
-#   packages => ['package1', 'package2', 'package3'],
-#
-class profiles::base::packages (
-  Array     $packages,
-  Enum[
-    'present',
-    'absent',
-    'latest',
-    'installed'
-    ]       $ensure = 'installed',
-){
-  ensure_packages($packages, {'ensure' => $ensure})
-}

site/profiles/manifests/git/git.pp

@@ -1,24 +0,0 @@
-# Class: profiles::git::git
-#
-# This class ensures that the Git package is installed.
-#
-# It uses the 'package' resource to manage the Git package,
-# and will ensure that it is installed. This class does not 
-# manage any configurations related to Git, it only ensures 
-# that the package is installed.
-#
-# The class does not take any parameters.
-#
-# Example usage:
-# --------------
-# To use this class, you simply need to declare it in your manifest:
-# 
-# include profiles::git::git
-#
-# You do not need to pass any parameters.
-#
-class profiles::git::git {
-  package { 'git':
-    ensure => installed,
-  }
-}

site/profiles/manifests/packages/base.pp

@@ -0,0 +1,21 @@
+# This class manages the installation of packages for the base profile
+#
+# Parameters:
+# - $packages: An array of package names to be installed (optional)
+# - $ensure: Enum of present, absent, latest or installed (optional)
+#
+# Example usage:
+# class { 'profiles::base::packages':
+#   packages => ['package1', 'package2', 'package3'],
+#
+class profiles::packages::base (
+  Array     $packages = lookup('profiles::packages::base', Array, 'first', []),
+  Enum[
+    'present',
+    'absent',
+    'latest',
+    'installed'
+    ]       $ensure = 'installed',
+){
+  ensure_packages($packages, {'ensure' => $ensure})
+}

site/profiles/manifests/packages/git.pp

@@ -0,0 +1,11 @@
+# installs git related packages
+#
+class profiles::packages::git (
+  Array[String] $packages = lookup('profiles::packages::git', Array, 'first', ['git']),
+) {
+  $packages.each |String $package| {
+    package { $package:
+      ensure => installed,
+    }
+  }
+}

site/profiles/manifests/packages/reposync.pp

@@ -0,0 +1,11 @@
+# installs reposync related packages
+#
+class profiles::packages::reposync (
+  Array[String] $packages = lookup('profiles::packages::reposync', Array, 'first', ['createrepo']),
+) {
+  $packages.each |String $package| {
+    package { $package:
+      ensure => installed,
+    }
+  }
+}

site/profiles/manifests/packages/selinux.pp

@@ -0,0 +1,11 @@
+# installs selinux related packages
+#
+class profiles::packages::selinux (
+  Array[String] $packages = lookup('profiles::packages::selinux', Array, 'first', ['policycoreutils']),
+) {
+  $packages.each |String $package| {
+    package { $package:
+      ensure => installed,
+    }
+  }
+}

site/profiles/manifests/puppet/enc.pp

@@ -39,7 +39,7 @@ class profiles::puppet::enc (
   Boolean $force = false,
 ) {
 
-  include profiles::git::git
+  include profiles::packages::git
 
   vcsrepo { '/opt/puppetlabs/enc':
     ensure   => latest,

site/profiles/manifests/puppet/r10k.pp

@@ -37,7 +37,7 @@ class profiles::puppet::r10k (
   String $r10k_repo,
 ){
 
-  include profiles::git::git
+  include profiles::packages::git
 
   vcsrepo { '/etc/puppetlabs/r10k':
     ensure   => latest,

site/profiles/manifests/reposync/autopromoter.pp

@@ -0,0 +1,105 @@
+# setup the autopromoter
+class profiles::reposync::autopromoter {
+
+  # Ensure the autopromoter script is present and executable
+  file { '/usr/local/bin/autopromoter':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('profiles/reposync/autopromoter.erb'),
+  }
+
+  # daily autopromote service/timer
+  $_daily_timer = @(EOT)
+  [Unit]
+  Description=autopromoter daily timer
+  [Timer]
+  OnCalendar=*-*-* 05:00:00
+  RandomizedDelaySec=1s
+  [Install]
+  WantedBy=timers.target
+  EOT
+
+  $_daily_service = @(EOT)
+  [Unit]
+  Description=autopromoter daily service
+  [Service]
+  Type=oneshot
+  ExecStart=/usr/local/bin/autopromoter daily
+  User=root
+  Group=root
+  PermissionsStartOnly=false
+  PrivateTmp=no
+  EOT
+
+  systemd::timer { 'autopromoter-daily.timer':
+    timer_content   => $_daily_timer,
+    service_content => $_daily_service,
+    active          => true,
+    enable          => true,
+    require         => File['/usr/local/bin/autopromoter'],
+  }
+
+  # weekly autopromote service/timer
+  $_weekly_timer = @(EOT)
+  [Unit]
+  Description=autopromoter weekly timer
+  [Timer]
+  OnCalendar=Sun *-*-* 05:05:00
+  RandomizedDelaySec=1s
+  [Install]
+  WantedBy=timers.target
+  EOT
+
+  $_weekly_service = @(EOT)
+  [Unit]
+  Description=autopromoter weekly service
+  [Service]
+  Type=oneshot
+  ExecStart=/usr/local/bin/autopromoter weekly
+  User=root
+  Group=root
+  PermissionsStartOnly=false
+  PrivateTmp=no
+  EOT
+
+  systemd::timer { 'autopromoter-weekly.timer':
+    timer_content   => $_weekly_timer,
+    service_content => $_weekly_service,
+    active          => true,
+    enable          => true,
+    require         => File['/usr/local/bin/autopromoter'],
+  }
+
+  # monthly autopromote service/timer
+  $_monthly_timer = @(EOT)
+  [Unit]
+  Description=autopromoter monthly timer
+  [Timer]
+  OnCalendar=*-*-01 05:10:00
+  RandomizedDelaySec=1s
+  [Install]
+  WantedBy=timers.target
+  EOT
+
+  $_monthly_service = @(EOT)
+  [Unit]
+  Description=autopromoter monthly service
+  [Service]
+  Type=oneshot
+  ExecStart=/usr/local/bin/autopromoter monthly
+  User=root
+  Group=root
+  PermissionsStartOnly=false
+  PrivateTmp=no
+  EOT
+
+  systemd::timer { 'autopromoter-monthly.timer':
+    timer_content   => $_monthly_timer,
+    service_content => $_monthly_service,
+    active          => true,
+    enable          => true,
+    require         => File['/usr/local/bin/autopromoter'],
+  }
+}

site/profiles/manifests/reposync/autosyncer.pp

@@ -0,0 +1,44 @@
+# setup the autosyncer
+class profiles::reposync::autosyncer {
+
+  # Ensure the autosyncer script is present and executable
+  file { '/usr/local/bin/autosyncer':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('profiles/reposync/autosyncer.erb'),
+    require => Class['profiles::packages::reposync'],
+  }
+
+  # daily autosyncr service/timer
+  $_timer = @(EOT)
+  [Unit]
+  Description=autosyncer timer
+  [Timer]
+  OnCalendar=*-*-* 03:00:00
+  RandomizedDelaySec=1s
+  [Install]
+  WantedBy=timers.target
+  EOT
+
+  $_service = @(EOT)
+  [Unit]
+  Description=autosyncer service
+  [Service]
+  Type=oneshot
+  ExecStart=/usr/local/bin/autosyncer
+  User=root
+  Group=root
+  PermissionsStartOnly=false
+  PrivateTmp=no
+  EOT
+
+  systemd::timer { 'autosyncer.timer':
+    timer_content   => $_timer,
+    service_content => $_service,
+    active          => true,
+    enable          => true,
+    require         => File['/usr/local/bin/autosyncer'],
+  }
+}

site/profiles/manifests/reposync/repos.pp

@@ -0,0 +1,46 @@
+# define to generate repositories in yum
+define profiles::reposync::repos (
+  String          $repository,
+  String          $description,
+  String          $osname,
+  String          $release,
+  Stdlib::HTTPUrl $baseurl,
+  Stdlib::HTTPUrl $gpgkey,
+  String          $arch = 'x86_64',
+  String          $repo_owner = 'root',
+  String          $repo_group = 'root',
+  Stdlib::Absolutepath $basepath = '/data/repos',
+){
+
+  $repos_name = downcase("${osname}-${release}-${repository}-${arch}")
+  $conf_file = "/etc/reposync/conf.d/${repos_name}.conf"
+
+  # Create the repository configuration
+  yumrepo { $repos_name:
+    ensure   => 'present',
+    descr    => $description,
+    baseurl  => $baseurl,
+    gpgkey   => $gpgkey,
+    target   => '/etc/yum.repos.d/reposync.repo',
+    enabled  => 0,
+    gpgcheck => 1,
+  }
+
+  # Ensure the repo dest path exists
+  file { "${basepath}/live/${repos_name}" :
+    ensure => 'directory',
+    owner  => $repo_owner,
+    group  => $repo_group,
+    mode   => '0755',
+  }
+
+  # Create the repo configuration file
+  file { $conf_file:
+    ensure  => file,
+    owner   => $repo_owner,
+    group   => $repo_group,
+    mode    => '0644',
+    content => template('profiles/reposync/repo_conf.erb'),
+    require => File['/etc/reposync/conf.d'],
+  }
+}

site/profiles/manifests/reposync/syncer.pp

@@ -0,0 +1,30 @@
+# setup a reposync syncer
+class profiles::reposync::syncer {
+
+  include profiles::packages::reposync
+  include profiles::reposync::autosyncer
+  include profiles::reposync::autopromoter
+  include profiles::reposync::webserver
+
+  # Ensure the reposync config path exists
+  file { '/etc/reposync':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+  file { '/etc/reposync/conf.d':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  # get a list of repos as a hash, and iterate through them
+  $repos = lookup('profiles::reposync::repos_list', {})
+  $repos.each | String $name, Hash $repo_hash | {
+    profiles::reposync::repos { $name:
+      * => $repo_hash,
+    }
+  }
+}

site/profiles/manifests/reposync/webserver.pp

@@ -0,0 +1,58 @@
+# setup a reposync webserver
+class profiles::reposync::webserver (
+  String  $www_root = '/data/repos/snap',
+  String  $nginx_vhost = 'repos.main.unkin.net',
+  Integer $nginx_port = 80,
+  Boolean $favicon = true,
+  Boolean $selinux = true,
+) {
+
+  class { 'nginx': }
+
+  # create the nginx vhost
+  nginx::resource::server { $nginx_vhost:
+    listen_port          => $nginx_port,
+    server_name          => [$nginx_vhost],
+    use_default_location => true,
+    access_log           => "/var/log/nginx/${nginx_vhost}_access.log",
+    error_log            => "/var/log/nginx/${nginx_vhost}_error.log",
+    www_root             => $www_root,
+    autoindex            => 'on',
+  }
+
+  if $favicon {
+    file { "${www_root}/favicon.ico":
+      ensure => 'file',
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0644',
+      source => 'puppet:///modules/profiles/reposync/favicon.ico',
+    }
+  }
+
+  if $selinux {
+
+    # include packages that are required
+    include profiles::packages::selinux
+
+    # set httpd_sys_content_t to all files under the www_root
+    selinux::fcontext { $www_root:
+      ensure   => 'present',
+      seltype  => 'httpd_sys_content_t',
+      pathspec => "${www_root}(/.*)?",
+    }
+
+    # make sure we can connect to port 80
+    selboolean { 'httpd_can_network_connect':
+      persistent => true,
+      value      => 'on',
+    }
+
+    exec { "restorecon_${www_root}":
+      path        => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'],
+      command     => "restorecon -Rv ${www_root}",
+      refreshonly => true,
+      subscribe   => Selinux::Fcontext[$www_root],
+    }
+  }
+}

site/profiles/templates/reposync/autopromoter.erb

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Function to create symlink for snapshots
+create_symlink() {
+  local osname="$1"
+  local release="$2"
+  local repository="$3"
+  local basepath="$4"
+  local label="$5" # 'monthly', 'weekly', or 'daily'
+  local date_format="$6" # Date format for finding the snapshot
+
+  # The path where snapshots are stored
+  local snap_path="${basepath}/snap/${osname}/${release}/${repository}-${date_format}"
+
+  # The target path for the symlink
+  local symlink_target="${basepath}/snap/${osname}/${release}/${repository}-${label}"
+
+  # Check if the source directory exists
+  if [[ -d "$snap_path" ]]; then
+    # Create the symlink, overwrite if it already exists
+    ln -sfn "$snap_path" "$symlink_target"
+    echo "Symlink created for $snap_path -> $symlink_target"
+  else
+    echo "Snapshot path does not exist: $snap_path"
+    return 1
+  fi
+}
+
+# Determine which snapshot to promote based on the passed argument
+case "$1" in
+  monthly)
+    promote_date=$(date --date="$(date +%Y%m01) -1 month" +%Y%m%d)
+    ;;
+  weekly)
+    promote_date=$(date --date="last Sunday" +%Y%m%d)
+    ;;
+  daily)
+    promote_date=$(date --date="yesterday" +%Y%m%d)
+    ;;
+  *)
+    echo "Usage: $0 {monthly|weekly|daily}"
+    exit 1
+    ;;
+esac
+
+# Call the function with appropriate arguments
+# Iterate over the repositories to create symlinks for each
+for conf in /etc/reposync/conf.d/*.conf; do
+  source "$conf"
+
+  # Create symlink based on the provided argument
+  create_symlink "$OSNAME" "$RELEASE" "$REPOSITORY" "$BASEPATH" "$1" "$promote_date"
+done

site/profiles/templates/reposync/autosyncer.erb

@@ -0,0 +1,97 @@
+#!/usr/bin/bash
+
+# Function to perform reposync
+perform_reposync() {
+  local reponame="$1"
+  local basepath="$2"
+
+  /usr/bin/dnf reposync \
+    --gpgcheck \
+    --delete \
+    --downloadcomps \
+    --download-metadata \
+    --remote-time \
+    --disablerepo="*" \
+    --enablerepo="${reponame}" \
+    --download-path="${basepath}/live"
+}
+
+# Function to download GPG keys
+download_gpg_key() {
+  local gpgkeyurl="$1"
+  local reponame="$2"
+  local basepath="$3"
+
+  # Extract filename from URL
+  local filename=$(basename "$gpgkeyurl")
+
+  # Download GPG key to the specified path with the filename from the URL
+  wget -q -O "${basepath}/live/${reponame}/${filename}" "$gpgkeyurl" || {
+    echo "Failed to download GPG key from $gpgkeyurl"
+  }
+}
+
+# Function to perform rsync with hard links
+perform_rsync() {
+  local source_path="$1"
+  local dest_path="$2"
+
+  # Create the destination directory if it doesn't exist
+  mkdir -p "$dest_path"
+
+  # Use rsync to create hard links to the files in the destination directory
+  rsync -a --link-dest="$source_path" "$source_path"/* "$dest_path"
+}
+
+create_repo_metadata() {
+  local basepath="${1}"
+  local osname="${2}"
+  local release="${3}"
+  local repository="${4}"
+  local current_date="${5}"
+
+  local repo_path="${basepath}/snap/${osname}/${release}/${repository}-${current_date}"
+
+  if [[ -d "$repo_path" ]]; then
+      echo "Running createrepo on ${repo_path}..."
+      createrepo --update "${repo_path}"
+      if [[ $? -eq 0 ]]; then
+          echo "Successfully created repository metadata for ${repository}"
+      else
+          echo "Failed to create repository metadata for ${repository}" >&2
+          return 1
+      fi
+  else
+      echo "The specified repository path does not exist: ${repo_path}" >&2
+      return 1
+  fi
+}
+
+# Current date in the required format
+DATE=$(date +%Y%m%d)
+
+# iterate over each configuration file
+for conf in /etc/reposync/conf.d/*.conf; do
+
+  # source the configuration to get the variables
+  source "$conf"
+
+  # Call the function to download the GPG key
+  download_gpg_key "$GPGKEYURL" "$REPONAME" "$BASEPATH"
+
+  # Call the reposync function
+  perform_reposync "$REPONAME" "$BASEPATH"
+
+  # Path for rsync source
+  live_path="${BASEPATH}/live/${REPONAME}"
+
+  # Path for rsync destination
+  snap_path="${BASEPATH}/snap/${OSNAME}/${RELEASE}/${REPOSITORY}-${DATE}/${ARCH}/os"
+
+  # Call the rsync function
+  perform_rsync "$live_path" "$snap_path"
+
+  # After syncing each repo, fix the repository metadata
+  create_repo_metadata "${BASEPATH}" "${OSNAME}" "${RELEASE}" "${REPOSITORY}" "${DATE}"
+
+done

site/profiles/templates/reposync/repo_conf.erb

@@ -0,0 +1,8 @@
+# <%= @osname %>-<%= @release %>-<%= @repository %> repository configuration
+REPOSITORY="<%= @repository %>"
+REPONAME="<%= @repos_name %>"
+OSNAME="<%= @osname %>"
+RELEASE="<%= @release %>"
+ARCH="<%= @arch %>"
+BASEPATH="<%= @basepath %>"
+GPGKEYURL="<%= @gpgkey %>"

site/roles/manifests/infra/packagerepo.pp

@@ -3,4 +3,5 @@ class roles::infra::packagerepo {
   include profiles::defaults
   include profiles::base
   include profiles::base::datavol
+  include profiles::reposync::syncer
 }