Merge pull request 'neoloc/droneci' (#148) from neoloc/droneci into develop

Reviewed-on: https://git.query.consul/unkinben/puppet-prod/pulls/148
2024-08-25 00:01:27 +10:00 · 2024-08-25 00:01:27 +10:00 · 0f755b231f
commit 0f755b231f
parent 9f937b2869 2912cbb68b
15 changed files with 288 additions and 0 deletions
--- a/1
+++ b/1
@ -18,6 +18,7 @@ mod 'puppetlabs-xinetd', '3.4.1'
 mod 'puppetlabs-haproxy', '8.0.0'
 mod 'puppetlabs-java', '10.1.2'
 mod 'puppetlabs-reboot', '5.0.0'
+mod 'puppetlabs-docker', '10.0.1'

 # puppet
 mod 'puppet-python', '7.0.0'
--- a/hieradata/roles/infra/droneci/runner.yaml
+++ b/hieradata/roles/infra/droneci/runner.yaml
@ -0,0 +1 @@
+---
--- a/hieradata/roles/infra/droneci/server.eyaml
+++ b/hieradata/roles/infra/droneci/server.eyaml
@ -0,0 +1,8 @@
+---
+
+droneci_server::gitea_client_secret: ENC[PKCS7,MIIBqQYJKoZIhvcNAQcDoIIBmjCCAZYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAYpW+TtZOKW7ZLuprdAeulVDV80rbmoWQoLZgo19UqXKHQzNoxPyY0ra4w8/fzXy1MUiMgtQkSBWe3MpqYK9F/6UOw6Q7GZCHZtVlwU/97YZ8oqeuI1x9wkyerfb9xYO2ywpEvjmpfRV01y4Cs3fUZ/lnvrwvsFJT96tgljANf9RDE+sh/FJvojQW1EAejgBrSfWBLaCQyody46rv6lP/mZY2XADYmDC3kklGHw78YK87CuTPcXD7up9EOdJ3mtgvwlkMYwbH8IeLUyA2IKYtPiB9wZPYX+YLNfkhQkIA+EeX6UlDxzJueWRPaaDFD1J5P0T8HTR18Pnt3erkiPAmWTBsBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB6ALrwgc4UyI5r4Xz2rpXYgECF2jGUym4hS4nYlpmc+u3CD8kvjTLZUu1rI2NlvB7i03nUhIExPJvfVGVcam+z4b8ulOxdXjju1Rx6MOcmC+K9]
+droneci_server::rpc_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAF8cEANj72ACYYdilP52Oz4EXVt+stx838tFVV4SFukoCTmN+kJ3GUUEM9x+oDvo17CsZhDzx5dX0JGo7N9MCLHsR4XKx3GSoAKvaSiD73TbzbdTNJgTIE1tRP9HNbJKizwWLo4lo+OUNtwnu0Z5dkMLYRHyvZ1hG24qmdXVn9DI06gVGQw9YXGmNy8AvA0BKnaHvUnE5XoLNVKZ4g1yvc/nkYpK6nLB+X4sZ96PRig7khiuFVvAfpg5c2iWnF13ljIajnG9uY12RyGaAGfH4l/d3UEyuHyQL4zzT8N7gGt8fvg7eNFx51TyEpVRFLyQ7dqq/lzn0moXVT35PQr9K3DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCGJ+rkxhqmyUUiIQLG7PnggDBPH9gyYAW5/XjDp5xd6GnuSt/WWOwOGxhg+1BoPzbV5o+Xufg5zyMVdYeI1MWD/u8=]
+droneci_server::cookie_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAKTWrB7Cca8RgFEeP46puzhVWOAiI6nB+m5+/sgz1qNnn6VgNh9Q6D26p9HISrMp4k60KVuGXrIbZYpkvKZmv4zHgK2et+50sr/F1anhDRX4rsmGWVV9n8VhaIJFAyQkW4de9YxV9LAgk0tWs1BgfXv4cV4+sDBv0OSVIFJDst3LUWpBR6lsiV9IifvNNUrdOHkdt5XjuL4JVmc0nkjetAg9HSvJ9VHYLIQagFHnTQp0FWluE1/ibGNc2kz7D4K7cPz2bBxRJWomckSUwgQK0NrGID4D13haZXhKXAO/8QpQOwPra8vD9FYrFenMeFLwdw43NzSr2g/W1ss5PekbWGjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBASshs2UwkGqK4ShFfXni7cgDCDCnEiqcfxIz4X/Bq71IpRKan3uQbHOewFqjNGqoR+1oWupjRxzNL39H9YF1a6i6s=]
+droneci_server::database_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEASu4C45TYWZKgIoyqC3YdwYYXn+T+ruP6oIvhYFJ5dxeZ+6HtWbRMViErvpPuWYfgs5qt6Zj9eLz2hqimFCvKiAvzANeZ9hkhw/jkpmvG8iXpDFw6x8QKcPJteRo896KSLiGiVlZfRbgQCGAqiEMw6y6M9CvfCLzE/mZ9gOKjSJVKiioAnXU2fyq0Y0M6g0iLRw0VXl2BVc4ORCnVECARQPo48T3U+TT39q0ar4mRO1AFO0VA5iDJ6/EMPBcH3ekKO/1dB1UbV6VkD3s9BAHGyL5a5Wr6ztg/5Yl6VBCXmECZqpCx8jx8KDUoaj1R/+I83YQxbw9ch76j79haIK6+jzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAh6MmfbUELaWbSNZ9/Dev2gDD/E3G/uYJGXNGl1+PIVwGmi0z2BTXNqg7ax/b/uF5Xc9ZtBSPiSxR6BPRXN3GleNo=]
+droneci_server::postgres_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANpDnrpratpuYheXFrN4nwRTauPm9rZz2ubDyJlcxmah+kOWqsWIeEkv5GuATlymfAx5UuHPOv3dJPCSK+YuyQY+kGW/8uEFM68QrNi38NdRqEpdXuPBe5+AmWxcjYK3mdJ4maEwsbbxtYJmD8TF6kskS2P/KhnIzYR5PPHZTaYbEf/W5Da3l+J5WnFYpStuLq+86yZokBAygFPI+y/Ic+zJIdhpzVdLyGuqxGLXZq7nNMrjuNyFPKkCj1BBpuJTMCS4oPKCUTlm5hIIeeC2pFREI0CMTV5siZB8NphobPNn/ZbJrcs9q75LtIa47pkFYRbmV4WPctCwZXg6jtMleuzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDHJaChyidZq/5FN5n+ASJWgDCqUcR/DG9e8AD7fRmTb5BZM8XQ77a1hUJoaCycnMQ/5UyKmqU/7fLPrsxCf2vZU1M=]
+droneci_server::redis_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAYMoZlVXHC1qfkDptPy3PyWGrUs5y9M9AOv5Vn1AK6DYViixhjwPZUCfwTONmt7CiBuqmkDOpR5isWFiBo/+TMeMXlM9C/D+Svc9tpeHLpVaXAYeoz5um2InyBFkLWSaUaWSftF9U7O5Nv/OiLIsd7nn4T8Dd21rfUiRfUN/2HPLMCs+mW15Az9XNOcvfm+kPXDAcB+ukHde0vvtYTZEFWjMdwJjjE43DiCmAoLTcpvQxdlclojotBpFeBLZs/F21FDQ4hvBUvPBMuZ1o7ImSP5fuomYSFK8etbD7JO5gg5BN59lGl1ljyR83phv6wmmqlzB8wQl0pjEpni9o7fa9hTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCKJrkPA78qx3wAu1eroIjRgDA72N9U0YZf1MzACcGSOMU5RGO242RwlIKUrnuYcdC0dKkjLXtP+yVpSKkX5WxcsDQ=]
--- a/hieradata/roles/infra/droneci/server.yaml
+++ b/hieradata/roles/infra/droneci/server.yaml
@ -0,0 +1,78 @@
+---
+# additional altnames
+profiles::pki::vault::alt_names:
+  - droneci.main.unkin.net
+  - droneci.service.consul
+  - droneci.query.consul
+  - "droneci.service.%{facts.country}-%{facts.region}.consul"
+
+profiles::ssh::sign::principals:
+  - droneci.main.unkin.net
+  - droneci.service.consul
+  - droneci.query.consul
+
+hiera_include:
+  - docker
+  - profiles::sql::postgresdb
+  - droneci
+
+docker::version: latest
+docker::curl_ensure: false
+
+profiles::sql::postgresdb::dbname: droneci
+profiles::sql::postgresdb::dbuser: droneci
+profiles::sql::postgresdb::dbpass: "%{hiera('droneci_server::postgres_password')}"
+profiles::sql::postgresdb::members_lookup: true
+profiles::sql::postgresdb::members_role: roles::infra::droneci::server
+
+droneci::ports:
+  - 80:80
+  - 443:443
+droneci::volumes:
+  - type=bind,source=/var/lib/drone,target=/data
+  - type=bind,source=/etc/pki/tls/vault/certificate.crt,target=/etc/pki/tls/vault/certificate.crt,readonly
+  - type=bind,source=/etc/pki/tls/vault/private.key,target=/etc/pki/tls/vault/private.key,readonly
+  - type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/pki/tls/certs/ca-bundle.crt,readonly
+  - type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/ssl/certs/ca-certificates.crt,readonly
+droneci::env_vars:
+  DRONE_GITEA_SERVER: https://git.query.consul
+  DRONE_GITEA_CLIENT_ID: 3f6abbd9-1838-4d22-8023-f9bd8cf27c82
+  DRONE_GITEA_CLIENT_SECRET: "%{hiera('droneci_server::gitea_client_secret')}"
+  DRONE_RPC_SECRET: "%{hiera('droneci_server::rpc_secret')}"
+  DRONE_SERVER_HOST: droneci.query.consul
+  DRONE_SERVER_PROTO: https
+  DRONE_TLS_CERT: /etc/pki/tls/vault/certificate.crt
+  DRONE_TLS_KEY: /etc/pki/tls/vault/private.key
+  DRONE_COOKIE_SECRET: "%{hiera('droneci_server::cookie_secret')}"
+  DRONE_COOKIE_TIMEOUT: 720h
+  DRONE_HTTP_SSL_REDIRECT: true
+  DRONE_HTTP_SSL_TEMPORARY_REDIRECT: true
+  DRONE_HTTP_SSL_HOST: droneci.query.consul
+  DRONE_LOGS_TEXT: true
+  DRONE_LOGS_PRETTY: true
+  DRONE_LOGS_COLOR: true
+  DRONE_DATABASE_SECRET: "%{hiera('droneci_server::database_secret')}"
+  DRONE_DATABASE_DRIVER: postgres
+  DRONE_DATABASE_DATASOURCE: "postgres://droneci:%{hiera('droneci_server::postgres_password')}@master.patroni-prod.service.au-syd1.consul:5432/droneci?sslmode=disable"
+  DRONE_REDIS_CONNECTION: "redis://%{hiera('droneci_server::redis_password')}@redis-master-prod.service.au-syd1.consul:6379/2"
+
+consul::services:
+  droneci:
+    service_name: 'droneci'
+    tags:
+      - 'drone'
+      - 'droneci'
+    address: "%{facts.networking.ip}"
+    port: 443
+    checks:
+      - id: 'droneci_https_check'
+        name: 'droneci HTTPS Check'
+        http: "https://%{facts.networking.fqdn}:443"
+        method: 'GET'
+        tls_skip_verify: true
+        interval: '10s'
+        timeout: '1s'
+profiles::consul::client::node_rules:
+  - resource: service
+    segment: droneci
+    disposition: write
--- a/hieradata/roles/infra/storage/consul.yaml
+++ b/hieradata/roles/infra/storage/consul.yaml
@ -89,3 +89,9 @@ profiles::consul::prepared_query::rules:
    service_failover_n: 3
    service_only_passing: true
    ttl: 10
+  droneci:
+    ensure: 'present'
+    service_name: 'droneci'
+    service_failover_n: 3
+    service_only_passing: true
+    ttl: 10
--- a/modules/droneci/manifests/init.pp
+++ b/modules/droneci/manifests/init.pp
@ -0,0 +1,24 @@
+class droneci (
+  Hash $env_vars = {},
+  String $docker_image = 'drone/drone:2',
+  Array[String] $ports = [],
+  Array[String] $volumes = [],
+  Stdlib::Absolutepath $env_file = '/etc/sysconfig/droneci',
+) {
+
+  # Create the environment file from a template
+  file { $env_file:
+    ensure  => file,
+    content => template('droneci/droneci_env.erb'),
+    mode    => '0644',
+  }
+
+  # Define the systemd service for Drone CI
+  systemd::unit_file { 'droneci.service':
+    ensure    => present,
+    content   => template('droneci/droneci_service.erb'),
+    enable    => true,
+    active    => true,
+    subscribe => File[$env_file],
+  }
+}
--- a/modules/droneci/templates/droneci_env.erb
+++ b/modules/droneci/templates/droneci_env.erb
@ -0,0 +1,3 @@
+<% @env_vars.each do |key, value| -%>
+<%= key.upcase %>=<%= value %>
+<% end -%>
--- a/modules/droneci/templates/droneci_service.erb
+++ b/modules/droneci/templates/droneci_service.erb
@ -0,0 +1,20 @@
+[Unit]
+Description=Drone CI Service
+After=docker.service
+Requires=docker.service
+
+[Service]
+ExecStart=/usr/bin/docker run --rm \
+  --name=drone \
+<% @ports.each do |port| -%>
+  -p <%= port %> \
+<% end -%>
+<% @volumes.each do |volume| -%>
+  --mount <%= volume %> \
+<% end -%>
+  --env-file <%= @env_file %> \
+  <%= @docker_image %>
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
--- a/site/profiles/manifests/sql/patroni.pp
+++ b/site/profiles/manifests/sql/patroni.pp
@ -84,6 +84,16 @@ class profiles::sql::patroni (
    ],
  }

+  $connect_settings = {
+
+  }
+
+  # collect exported resources
+  $tag = "${facts['country']}-${facts['region']}-${facts['environment']}"
+  Profiles::Sql::Postgres::Db <<| tag == $tag |>> {}
+  Profiles::Sql::Postgres::User <<| tag == $tag |>> {}
+  Profiles::Sql::Postgres::Grant <<| tag == $tag |>> {}
+
  if $postgres_exporter_enabled {
    class { 'prometheus::postgres_exporter':
      postgres_user     => $postgres_exporter_user,
--- a/site/profiles/manifests/sql/postgres/db.pp
+++ b/site/profiles/manifests/sql/postgres/db.pp
@ -0,0 +1,9 @@
+define profiles::sql::postgres::db (
+  String $dbname,
+) {
+  postgresql_psql { "create_database_${dbname}":
+    command => "CREATE DATABASE \"${dbname}\"",
+    unless  => "SELECT 1 FROM pg_database WHERE datname = '${dbname}'",
+  }
+}
+
--- a/site/profiles/manifests/sql/postgres/grant.pp
+++ b/site/profiles/manifests/sql/postgres/grant.pp
@ -0,0 +1,38 @@
+define profiles::sql::postgres::grant (
+  String $username,
+  Enum['SCHEMA', 'DATABASE'] $type = 'DATABASE',
+  Optional[String] $dbname = undef,
+  Optional[String] $schema = undef,
+  String $privilege = 'ALL PRIVILEGES',
+) {
+  # Validate parameters based on type
+  if $type == 'DATABASE' and $dbname == undef {
+    fail('The dbname parameter must be provided when type is DATABASE')
+  }
+
+  if $type == 'SCHEMA' and ($dbname == undef or $schema == undef) {
+    fail('Both dbname and schema parameters must be provided when type is SCHEMA')
+  }
+
+  # Determine the appropriate SQL command and unless condition
+  $command = $type ? {
+    'DATABASE' => "GRANT ${privilege} ON DATABASE ${dbname} TO ${username}",
+    'SCHEMA'   => "GRANT ${privilege} ON SCHEMA ${schema} TO ${username}",
+  }
+
+  $unless = $type ? {
+    'DATABASE' => "SELECT 1 FROM pg_roles r WHERE r.rolname='${username}' AND has_database_privilege('${username}', '${dbname}', 'CONNECT')", # lint:ignore:140chars
+    'SCHEMA'   => "SELECT 1 FROM pg_namespace n JOIN pg_roles r ON r.oid = n.nspowner WHERE nspname = '${schema}' AND r.rolname = '${username}'", # lint:ignore:140chars
+  }
+  # Ensure the db parameter is set correctly when type is SCHEMA
+  $effective_dbname = $type ? {
+    'SCHEMA'   => $dbname,
+    'DATABASE' => $dbname,
+  }
+
+  postgresql_psql { "grant_${privilege}_on_${type}_${effective_dbname}_${schema}_to_${username}":
+    command => $command,
+    unless  => $unless,
+    db      => $effective_dbname,
+  }
+}
--- a/site/profiles/manifests/sql/postgres/user.pp
+++ b/site/profiles/manifests/sql/postgres/user.pp
@ -0,0 +1,9 @@
+define profiles::sql::postgres::user (
+  String $username,
+  String $password,
+) {
+  postgresql_psql { "create_user_${username}":
+    command => "CREATE USER \"${username}\" WITH ENCRYPTED PASSWORD '${password}'",
+    unless  => "SELECT 1 FROM pg_roles WHERE rolname = '${username}'",
+  }
+}
--- a/site/profiles/manifests/sql/postgresdb.pp
+++ b/site/profiles/manifests/sql/postgresdb.pp
@ -0,0 +1,61 @@
+class profiles::sql::postgresdb (
+  String  $dbname,
+  String  $dbuser,
+  String  $dbpass,
+  Boolean $create_host_users = false,
+  Boolean $members_lookup    = false,
+  String  $members_role      = undef,
+  Array   $servers           = [],
+){
+
+  # if lookup is enabled
+  if $members_lookup {
+
+    # check that the role is also set
+    unless !($members_role == undef) {
+      fail("members_role must be provided for ${title} when members_lookup is True")
+    }
+
+    # if it is, find hosts, sort them so they dont cause changes every run
+    $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn'))
+
+  # else use provided array from params
+  }else{
+    $servers_array = $servers
+  }
+
+  $tag = "${facts['country']}-${facts['region']}-${facts['environment']}"
+
+  # only export from the first server in a cluster
+  if $servers_array[0] == $facts['networking']['fqdn'] {
+
+    # manage the postgres db
+    @@profiles::sql::postgres::db { "${facts['networking']['fqdn']}_db_${dbname}":
+      dbname => $dbname,
+      tag    => $tag,
+    }
+
+    @@profiles::sql::postgres::user { "${facts['networking']['fqdn']}_role_${dbuser}":
+      username => $dbuser,
+      password => $dbpass,
+      tag      => $tag,
+    }
+
+    @@profiles::sql::postgres::grant { "${facts['networking']['fqdn']}_grant_db_${dbuser}_${dbuser}}":
+      dbname    => $dbname,
+      username  => $dbuser,
+      type      => 'DATABASE',
+      privilege => 'ALL PRIVILEGES',
+      tag       => $tag,
+    }
+
+    @@profiles::sql::postgres::grant { "${facts['networking']['fqdn']}_grant_schema_${dbuser}_${dbuser}}":
+      dbname    => $dbname,
+      username  => $dbuser,
+      type      => 'SCHEMA',
+      schema    => 'public',
+      privilege => 'ALL PRIVILEGES',
+      tag       => $tag,
+    }
+  }
+}
--- a/site/roles/manifests/infra/droneci/runner.pp
+++ b/site/roles/manifests/infra/droneci/runner.pp
@ -0,0 +1,10 @@
+# a role to deploy droneci
+class roles::infra::droneci::runner {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}
--- a/site/roles/manifests/infra/droneci/server.pp
+++ b/site/roles/manifests/infra/droneci/server.pp
@ -0,0 +1,10 @@
+# a role to deploy droneci
+class roles::infra::droneci::server {
+  if $facts['firstrun'] {
+    include profiles::defaults
+    include profiles::firstrun::init
+  }else{
+    include profiles::defaults
+    include profiles::base
+  }
+}