diff --git a/Puppetfile b/Puppetfile index 342aa42..ddb6e0a 100644 --- a/Puppetfile +++ b/Puppetfile @@ -18,6 +18,7 @@ mod 'puppetlabs-xinetd', '3.4.1' mod 'puppetlabs-haproxy', '8.0.0' mod 'puppetlabs-java', '10.1.2' mod 'puppetlabs-reboot', '5.0.0' +mod 'puppetlabs-docker', '10.0.1' # puppet mod 'puppet-python', '7.0.0' diff --git a/hieradata/roles/infra/droneci/runner.yaml b/hieradata/roles/infra/droneci/runner.yaml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/hieradata/roles/infra/droneci/runner.yaml @@ -0,0 +1 @@ +--- diff --git a/hieradata/roles/infra/droneci/server.eyaml b/hieradata/roles/infra/droneci/server.eyaml new file mode 100644 index 0000000..25bfb17 --- /dev/null +++ b/hieradata/roles/infra/droneci/server.eyaml @@ -0,0 +1,8 @@ +--- + +droneci_server::gitea_client_secret: ENC[PKCS7,MIIBqQYJKoZIhvcNAQcDoIIBmjCCAZYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAYpW+TtZOKW7ZLuprdAeulVDV80rbmoWQoLZgo19UqXKHQzNoxPyY0ra4w8/fzXy1MUiMgtQkSBWe3MpqYK9F/6UOw6Q7GZCHZtVlwU/97YZ8oqeuI1x9wkyerfb9xYO2ywpEvjmpfRV01y4Cs3fUZ/lnvrwvsFJT96tgljANf9RDE+sh/FJvojQW1EAejgBrSfWBLaCQyody46rv6lP/mZY2XADYmDC3kklGHw78YK87CuTPcXD7up9EOdJ3mtgvwlkMYwbH8IeLUyA2IKYtPiB9wZPYX+YLNfkhQkIA+EeX6UlDxzJueWRPaaDFD1J5P0T8HTR18Pnt3erkiPAmWTBsBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBB6ALrwgc4UyI5r4Xz2rpXYgECF2jGUym4hS4nYlpmc+u3CD8kvjTLZUu1rI2NlvB7i03nUhIExPJvfVGVcam+z4b8ulOxdXjju1Rx6MOcmC+K9] +droneci_server::rpc_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAF8cEANj72ACYYdilP52Oz4EXVt+stx838tFVV4SFukoCTmN+kJ3GUUEM9x+oDvo17CsZhDzx5dX0JGo7N9MCLHsR4XKx3GSoAKvaSiD73TbzbdTNJgTIE1tRP9HNbJKizwWLo4lo+OUNtwnu0Z5dkMLYRHyvZ1hG24qmdXVn9DI06gVGQw9YXGmNy8AvA0BKnaHvUnE5XoLNVKZ4g1yvc/nkYpK6nLB+X4sZ96PRig7khiuFVvAfpg5c2iWnF13ljIajnG9uY12RyGaAGfH4l/d3UEyuHyQL4zzT8N7gGt8fvg7eNFx51TyEpVRFLyQ7dqq/lzn0moXVT35PQr9K3DBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCGJ+rkxhqmyUUiIQLG7PnggDBPH9gyYAW5/XjDp5xd6GnuSt/WWOwOGxhg+1BoPzbV5o+Xufg5zyMVdYeI1MWD/u8=] +droneci_server::cookie_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAKTWrB7Cca8RgFEeP46puzhVWOAiI6nB+m5+/sgz1qNnn6VgNh9Q6D26p9HISrMp4k60KVuGXrIbZYpkvKZmv4zHgK2et+50sr/F1anhDRX4rsmGWVV9n8VhaIJFAyQkW4de9YxV9LAgk0tWs1BgfXv4cV4+sDBv0OSVIFJDst3LUWpBR6lsiV9IifvNNUrdOHkdt5XjuL4JVmc0nkjetAg9HSvJ9VHYLIQagFHnTQp0FWluE1/ibGNc2kz7D4K7cPz2bBxRJWomckSUwgQK0NrGID4D13haZXhKXAO/8QpQOwPra8vD9FYrFenMeFLwdw43NzSr2g/W1ss5PekbWGjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBASshs2UwkGqK4ShFfXni7cgDCDCnEiqcfxIz4X/Bq71IpRKan3uQbHOewFqjNGqoR+1oWupjRxzNL39H9YF1a6i6s=] +droneci_server::database_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEASu4C45TYWZKgIoyqC3YdwYYXn+T+ruP6oIvhYFJ5dxeZ+6HtWbRMViErvpPuWYfgs5qt6Zj9eLz2hqimFCvKiAvzANeZ9hkhw/jkpmvG8iXpDFw6x8QKcPJteRo896KSLiGiVlZfRbgQCGAqiEMw6y6M9CvfCLzE/mZ9gOKjSJVKiioAnXU2fyq0Y0M6g0iLRw0VXl2BVc4ORCnVECARQPo48T3U+TT39q0ar4mRO1AFO0VA5iDJ6/EMPBcH3ekKO/1dB1UbV6VkD3s9BAHGyL5a5Wr6ztg/5Yl6VBCXmECZqpCx8jx8KDUoaj1R/+I83YQxbw9ch76j79haIK6+jzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAh6MmfbUELaWbSNZ9/Dev2gDD/E3G/uYJGXNGl1+PIVwGmi0z2BTXNqg7ax/b/uF5Xc9ZtBSPiSxR6BPRXN3GleNo=] +droneci_server::postgres_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEANpDnrpratpuYheXFrN4nwRTauPm9rZz2ubDyJlcxmah+kOWqsWIeEkv5GuATlymfAx5UuHPOv3dJPCSK+YuyQY+kGW/8uEFM68QrNi38NdRqEpdXuPBe5+AmWxcjYK3mdJ4maEwsbbxtYJmD8TF6kskS2P/KhnIzYR5PPHZTaYbEf/W5Da3l+J5WnFYpStuLq+86yZokBAygFPI+y/Ic+zJIdhpzVdLyGuqxGLXZq7nNMrjuNyFPKkCj1BBpuJTMCS4oPKCUTlm5hIIeeC2pFREI0CMTV5siZB8NphobPNn/ZbJrcs9q75LtIa47pkFYRbmV4WPctCwZXg6jtMleuzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBDHJaChyidZq/5FN5n+ASJWgDCqUcR/DG9e8AD7fRmTb5BZM8XQ77a1hUJoaCycnMQ/5UyKmqU/7fLPrsxCf2vZU1M=] +droneci_server::redis_password: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAYMoZlVXHC1qfkDptPy3PyWGrUs5y9M9AOv5Vn1AK6DYViixhjwPZUCfwTONmt7CiBuqmkDOpR5isWFiBo/+TMeMXlM9C/D+Svc9tpeHLpVaXAYeoz5um2InyBFkLWSaUaWSftF9U7O5Nv/OiLIsd7nn4T8Dd21rfUiRfUN/2HPLMCs+mW15Az9XNOcvfm+kPXDAcB+ukHde0vvtYTZEFWjMdwJjjE43DiCmAoLTcpvQxdlclojotBpFeBLZs/F21FDQ4hvBUvPBMuZ1o7ImSP5fuomYSFK8etbD7JO5gg5BN59lGl1ljyR83phv6wmmqlzB8wQl0pjEpni9o7fa9hTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCKJrkPA78qx3wAu1eroIjRgDA72N9U0YZf1MzACcGSOMU5RGO242RwlIKUrnuYcdC0dKkjLXtP+yVpSKkX5WxcsDQ=] diff --git a/hieradata/roles/infra/droneci/server.yaml b/hieradata/roles/infra/droneci/server.yaml new file mode 100644 index 0000000..74aab53 --- /dev/null +++ b/hieradata/roles/infra/droneci/server.yaml @@ -0,0 +1,78 @@ +--- +# additional altnames +profiles::pki::vault::alt_names: + - droneci.main.unkin.net + - droneci.service.consul + - droneci.query.consul + - "droneci.service.%{facts.country}-%{facts.region}.consul" + +profiles::ssh::sign::principals: + - droneci.main.unkin.net + - droneci.service.consul + - droneci.query.consul + +hiera_include: + - docker + - profiles::sql::postgresdb + - droneci + +docker::version: latest +docker::curl_ensure: false + +profiles::sql::postgresdb::dbname: droneci +profiles::sql::postgresdb::dbuser: droneci +profiles::sql::postgresdb::dbpass: "%{hiera('droneci_server::postgres_password')}" +profiles::sql::postgresdb::members_lookup: true +profiles::sql::postgresdb::members_role: roles::infra::droneci::server + +droneci::ports: + - 80:80 + - 443:443 +droneci::volumes: + - type=bind,source=/var/lib/drone,target=/data + - type=bind,source=/etc/pki/tls/vault/certificate.crt,target=/etc/pki/tls/vault/certificate.crt,readonly + - type=bind,source=/etc/pki/tls/vault/private.key,target=/etc/pki/tls/vault/private.key,readonly + - type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/pki/tls/certs/ca-bundle.crt,readonly + - type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/ssl/certs/ca-certificates.crt,readonly +droneci::env_vars: + DRONE_GITEA_SERVER: https://git.query.consul + DRONE_GITEA_CLIENT_ID: 3f6abbd9-1838-4d22-8023-f9bd8cf27c82 + DRONE_GITEA_CLIENT_SECRET: "%{hiera('droneci_server::gitea_client_secret')}" + DRONE_RPC_SECRET: "%{hiera('droneci_server::rpc_secret')}" + DRONE_SERVER_HOST: droneci.query.consul + DRONE_SERVER_PROTO: https + DRONE_TLS_CERT: /etc/pki/tls/vault/certificate.crt + DRONE_TLS_KEY: /etc/pki/tls/vault/private.key + DRONE_COOKIE_SECRET: "%{hiera('droneci_server::cookie_secret')}" + DRONE_COOKIE_TIMEOUT: 720h + DRONE_HTTP_SSL_REDIRECT: true + DRONE_HTTP_SSL_TEMPORARY_REDIRECT: true + DRONE_HTTP_SSL_HOST: droneci.query.consul + DRONE_LOGS_TEXT: true + DRONE_LOGS_PRETTY: true + DRONE_LOGS_COLOR: true + DRONE_DATABASE_SECRET: "%{hiera('droneci_server::database_secret')}" + DRONE_DATABASE_DRIVER: postgres + DRONE_DATABASE_DATASOURCE: "postgres://droneci:%{hiera('droneci_server::postgres_password')}@master.patroni-prod.service.au-syd1.consul:5432/droneci?sslmode=disable" + DRONE_REDIS_CONNECTION: "redis://%{hiera('droneci_server::redis_password')}@redis-master-prod.service.au-syd1.consul:6379/2" + +consul::services: + droneci: + service_name: 'droneci' + tags: + - 'drone' + - 'droneci' + address: "%{facts.networking.ip}" + port: 443 + checks: + - id: 'droneci_https_check' + name: 'droneci HTTPS Check' + http: "https://%{facts.networking.fqdn}:443" + method: 'GET' + tls_skip_verify: true + interval: '10s' + timeout: '1s' +profiles::consul::client::node_rules: + - resource: service + segment: droneci + disposition: write diff --git a/hieradata/roles/infra/storage/consul.yaml b/hieradata/roles/infra/storage/consul.yaml index 53abf8f..148d2f0 100644 --- a/hieradata/roles/infra/storage/consul.yaml +++ b/hieradata/roles/infra/storage/consul.yaml @@ -89,3 +89,9 @@ profiles::consul::prepared_query::rules: service_failover_n: 3 service_only_passing: true ttl: 10 + droneci: + ensure: 'present' + service_name: 'droneci' + service_failover_n: 3 + service_only_passing: true + ttl: 10 diff --git a/modules/droneci/manifests/init.pp b/modules/droneci/manifests/init.pp new file mode 100644 index 0000000..cba7d1f --- /dev/null +++ b/modules/droneci/manifests/init.pp @@ -0,0 +1,24 @@ +class droneci ( + Hash $env_vars = {}, + String $docker_image = 'drone/drone:2', + Array[String] $ports = [], + Array[String] $volumes = [], + Stdlib::Absolutepath $env_file = '/etc/sysconfig/droneci', +) { + + # Create the environment file from a template + file { $env_file: + ensure => file, + content => template('droneci/droneci_env.erb'), + mode => '0644', + } + + # Define the systemd service for Drone CI + systemd::unit_file { 'droneci.service': + ensure => present, + content => template('droneci/droneci_service.erb'), + enable => true, + active => true, + subscribe => File[$env_file], + } +} diff --git a/modules/droneci/templates/droneci_env.erb b/modules/droneci/templates/droneci_env.erb new file mode 100644 index 0000000..47659c8 --- /dev/null +++ b/modules/droneci/templates/droneci_env.erb @@ -0,0 +1,3 @@ +<% @env_vars.each do |key, value| -%> +<%= key.upcase %>=<%= value %> +<% end -%> diff --git a/modules/droneci/templates/droneci_service.erb b/modules/droneci/templates/droneci_service.erb new file mode 100644 index 0000000..6cc46e0 --- /dev/null +++ b/modules/droneci/templates/droneci_service.erb @@ -0,0 +1,20 @@ +[Unit] +Description=Drone CI Service +After=docker.service +Requires=docker.service + +[Service] +ExecStart=/usr/bin/docker run --rm \ + --name=drone \ +<% @ports.each do |port| -%> + -p <%= port %> \ +<% end -%> +<% @volumes.each do |volume| -%> + --mount <%= volume %> \ +<% end -%> + --env-file <%= @env_file %> \ + <%= @docker_image %> +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/site/profiles/manifests/sql/patroni.pp b/site/profiles/manifests/sql/patroni.pp index 69449cb..e53dcc4 100644 --- a/site/profiles/manifests/sql/patroni.pp +++ b/site/profiles/manifests/sql/patroni.pp @@ -84,6 +84,16 @@ class profiles::sql::patroni ( ], } + $connect_settings = { + + } + + # collect exported resources + $tag = "${facts['country']}-${facts['region']}-${facts['environment']}" + Profiles::Sql::Postgres::Db <<| tag == $tag |>> {} + Profiles::Sql::Postgres::User <<| tag == $tag |>> {} + Profiles::Sql::Postgres::Grant <<| tag == $tag |>> {} + if $postgres_exporter_enabled { class { 'prometheus::postgres_exporter': postgres_user => $postgres_exporter_user, diff --git a/site/profiles/manifests/sql/postgres/db.pp b/site/profiles/manifests/sql/postgres/db.pp new file mode 100644 index 0000000..586820e --- /dev/null +++ b/site/profiles/manifests/sql/postgres/db.pp @@ -0,0 +1,9 @@ +define profiles::sql::postgres::db ( + String $dbname, +) { + postgresql_psql { "create_database_${dbname}": + command => "CREATE DATABASE \"${dbname}\"", + unless => "SELECT 1 FROM pg_database WHERE datname = '${dbname}'", + } +} + diff --git a/site/profiles/manifests/sql/postgres/grant.pp b/site/profiles/manifests/sql/postgres/grant.pp new file mode 100644 index 0000000..2f8476e --- /dev/null +++ b/site/profiles/manifests/sql/postgres/grant.pp @@ -0,0 +1,38 @@ +define profiles::sql::postgres::grant ( + String $username, + Enum['SCHEMA', 'DATABASE'] $type = 'DATABASE', + Optional[String] $dbname = undef, + Optional[String] $schema = undef, + String $privilege = 'ALL PRIVILEGES', +) { + # Validate parameters based on type + if $type == 'DATABASE' and $dbname == undef { + fail('The dbname parameter must be provided when type is DATABASE') + } + + if $type == 'SCHEMA' and ($dbname == undef or $schema == undef) { + fail('Both dbname and schema parameters must be provided when type is SCHEMA') + } + + # Determine the appropriate SQL command and unless condition + $command = $type ? { + 'DATABASE' => "GRANT ${privilege} ON DATABASE ${dbname} TO ${username}", + 'SCHEMA' => "GRANT ${privilege} ON SCHEMA ${schema} TO ${username}", + } + + $unless = $type ? { + 'DATABASE' => "SELECT 1 FROM pg_roles r WHERE r.rolname='${username}' AND has_database_privilege('${username}', '${dbname}', 'CONNECT')", # lint:ignore:140chars + 'SCHEMA' => "SELECT 1 FROM pg_namespace n JOIN pg_roles r ON r.oid = n.nspowner WHERE nspname = '${schema}' AND r.rolname = '${username}'", # lint:ignore:140chars + } + # Ensure the db parameter is set correctly when type is SCHEMA + $effective_dbname = $type ? { + 'SCHEMA' => $dbname, + 'DATABASE' => $dbname, + } + + postgresql_psql { "grant_${privilege}_on_${type}_${effective_dbname}_${schema}_to_${username}": + command => $command, + unless => $unless, + db => $effective_dbname, + } +} diff --git a/site/profiles/manifests/sql/postgres/user.pp b/site/profiles/manifests/sql/postgres/user.pp new file mode 100644 index 0000000..b97ba49 --- /dev/null +++ b/site/profiles/manifests/sql/postgres/user.pp @@ -0,0 +1,9 @@ +define profiles::sql::postgres::user ( + String $username, + String $password, +) { + postgresql_psql { "create_user_${username}": + command => "CREATE USER \"${username}\" WITH ENCRYPTED PASSWORD '${password}'", + unless => "SELECT 1 FROM pg_roles WHERE rolname = '${username}'", + } +} diff --git a/site/profiles/manifests/sql/postgresdb.pp b/site/profiles/manifests/sql/postgresdb.pp new file mode 100644 index 0000000..66b5268 --- /dev/null +++ b/site/profiles/manifests/sql/postgresdb.pp @@ -0,0 +1,61 @@ +class profiles::sql::postgresdb ( + String $dbname, + String $dbuser, + String $dbpass, + Boolean $create_host_users = false, + Boolean $members_lookup = false, + String $members_role = undef, + Array $servers = [], +){ + + # if lookup is enabled + if $members_lookup { + + # check that the role is also set + unless !($members_role == undef) { + fail("members_role must be provided for ${title} when members_lookup is True") + } + + # if it is, find hosts, sort them so they dont cause changes every run + $servers_array = sort(query_nodes("enc_role='${members_role}' and region='${facts['region']}'", 'networking.fqdn')) + + # else use provided array from params + }else{ + $servers_array = $servers + } + + $tag = "${facts['country']}-${facts['region']}-${facts['environment']}" + + # only export from the first server in a cluster + if $servers_array[0] == $facts['networking']['fqdn'] { + + # manage the postgres db + @@profiles::sql::postgres::db { "${facts['networking']['fqdn']}_db_${dbname}": + dbname => $dbname, + tag => $tag, + } + + @@profiles::sql::postgres::user { "${facts['networking']['fqdn']}_role_${dbuser}": + username => $dbuser, + password => $dbpass, + tag => $tag, + } + + @@profiles::sql::postgres::grant { "${facts['networking']['fqdn']}_grant_db_${dbuser}_${dbuser}}": + dbname => $dbname, + username => $dbuser, + type => 'DATABASE', + privilege => 'ALL PRIVILEGES', + tag => $tag, + } + + @@profiles::sql::postgres::grant { "${facts['networking']['fqdn']}_grant_schema_${dbuser}_${dbuser}}": + dbname => $dbname, + username => $dbuser, + type => 'SCHEMA', + schema => 'public', + privilege => 'ALL PRIVILEGES', + tag => $tag, + } + } +} diff --git a/site/roles/manifests/infra/droneci/runner.pp b/site/roles/manifests/infra/droneci/runner.pp new file mode 100644 index 0000000..6e76a5a --- /dev/null +++ b/site/roles/manifests/infra/droneci/runner.pp @@ -0,0 +1,10 @@ +# a role to deploy droneci +class roles::infra::droneci::runner { + if $facts['firstrun'] { + include profiles::defaults + include profiles::firstrun::init + }else{ + include profiles::defaults + include profiles::base + } +} diff --git a/site/roles/manifests/infra/droneci/server.pp b/site/roles/manifests/infra/droneci/server.pp new file mode 100644 index 0000000..e015186 --- /dev/null +++ b/site/roles/manifests/infra/droneci/server.pp @@ -0,0 +1,10 @@ +# a role to deploy droneci +class roles::infra::droneci::server { + if $facts['firstrun'] { + include profiles::defaults + include profiles::firstrun::init + }else{ + include profiles::defaults + include profiles::base + } +}