diff --git a/README.md b/README.md index b2069fb..cc64429 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,17 @@ The BinaryLane API token is read from `kv/service/terraform/binarylane` (field from `consul_root/au/syd1/creds/terraform-binarylane`. Both are granted to the `terraform_binarylane` AppRole and the `woodpecker_terraform_binarylane` k8s role in terraform-vault. + +## Mail edge + +Each DMZ VPC (`syd` 198.18.30.0/24, `mel` 198.18.31.0/24) holds a postfix edge host and a vpn host. The vpn +host runs wg0 as a /31 point-to-point link to its own prodnxsr0020 interface +(`wg-syd` on 51821, `wg-mel` on 51822; `router_links` output) and forwards the +lab routes for its VPC; the edge host reaches the lab through it and has no +WireGuard. The VPC /24 is connected on both hosts, so it wins over the lab +198.18.0.0/16 route via the vpn host. Link /31s and vpn loopbacks (dum0) come from 198.18.2.160/27. Lab +traffic is sourced from the /31 until Puppet brings up FRR, which peers OSPF +over wg0 and advertises the loopback and VPC to the router. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-`. +wg0 is bootstrapped from a key in `user_data` (readable via the BinaryLane +API); Puppet then manages it from Vault, so rotate by replacing +`wireguard_asymmetric_key.vpn` here and letting Puppet roll it out. diff --git a/modules/binarylane/.terraform.lock.hcl b/modules/binarylane/.terraform.lock.hcl new file mode 100644 index 0000000..39ba811 --- /dev/null +++ b/modules/binarylane/.terraform.lock.hcl @@ -0,0 +1,50 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" { + version = "0.14.0" + constraints = "0.14.0" + hashes = [ + "h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=", + "zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45", + ] +} + +provider "registry.opentofu.org/hashicorp/vault" { + version = "5.6.0" + constraints = "5.6.0" + hashes = [ + "h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=", + "zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8", + "zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a", + "zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8", + "zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7", + "zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf", + "zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95", + "zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d", + "zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21", + "zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b", + ] +} + +provider "registry.opentofu.org/ojford/wireguard" { + version = "0.4.1" + constraints = "0.4.1" + hashes = [ + "h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620", + "zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1", + "zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46", + "zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7", + "zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2", + "zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2", + "zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92", + "zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1", + "zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863", + "zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b", + "zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e", + "zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37", + "zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1", + ] +} diff --git a/modules/binarylane/.tflint.hcl b/modules/binarylane/.tflint.hcl deleted file mode 100644 index 3a2ace1..0000000 --- a/modules/binarylane/.tflint.hcl +++ /dev/null @@ -1,3 +0,0 @@ -rule "terraform_unused_declarations" { - enabled = false -} diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl new file mode 100644 index 0000000..3dbab72 --- /dev/null +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -0,0 +1,100 @@ +#cloud-config +hostname: ${hostname} +fqdn: ${fqdn} +prefer_fqdn_over_hostname: true + +write_files: +%{ if vpn != null ~} + - path: /etc/wireguard/wg0.conf + permissions: "0600" + content: | + [Interface] + Address = ${vpn.address} + ListenPort = ${vpn.port} + PrivateKey = ${vpn.private_key} + # source lab traffic from the /31: the router only learns the loopback and vpc once ospf is up + Table = off +%{ for r in vpn.routes ~} + PostUp = ip route replace ${r} dev wg0 src ${vpn.src} +%{ endfor ~} + + [Peer] + PublicKey = ${vpn.router_key} + Endpoint = ${vpn.router} + AllowedIPs = ${vpn.allowed_ips} + PersistentKeepalive = 25 + - path: /etc/NetworkManager/system-connections/dum0.nmconnection + permissions: "0600" + content: | + [connection] + id=dum0 + type=dummy + interface-name=dum0 + + [ipv4] + method=manual + address1=${vpn.loopback} + + [ipv6] + method=disabled + - path: /etc/sysctl.d/90-ip-forward.conf + content: | + net.ipv4.ip_forward = 1 +%{ endif ~} + - path: /usr/local/sbin/lab-bootstrap + permissions: "0755" + content: | + #!/bin/bash + set -euo pipefail + if ! rpm -q openvox-agent >/dev/null; then + rpm -q openvox7-release >/dev/null || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm + dnf install -y openvox-agent + fi + /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn} + /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main environment production +%{ if vpn != null ~} + rpm -q wireguard-tools >/dev/null || dnf install -y wireguard-tools + sysctl -p /etc/sysctl.d/90-ip-forward.conf + nmcli connection reload + nmcli connection up dum0 + systemctl enable --now wg-quick@wg0 + wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }' +%{ endif ~} +%{ if routes != null ~} + if ! ip route show 198.18.200.3 | grep -q .; then + dev=$(ip -o -4 addr show to ${routes.vpc} | awk '{ print $2; exit }') + con=$(nmcli -g GENERAL.CONNECTION device show "$dev") + nmcli connection modify "$con" ipv4.routes "${routes.list}" + nmcli device reapply "$dev" + fi +%{ endif ~} + timeout 5 bash -c ' /etc/NetworkManager/conf.d/90-dns-none.conf + nmcli general reload + rm -f /etc/resolv.conf + printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf + systemctl enable --now puppet + systemctl disable lab-bootstrap + - path: /etc/systemd/system/lab-bootstrap.service + content: | + [Unit] + Description=Install openvox, wait for the lab via the vpn host, then switch to internal DNS and start puppet + Wants=network-online.target + After=network-online.target + StartLimitIntervalSec=0 + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/lab-bootstrap + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable --now --no-block lab-bootstrap diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf new file mode 100644 index 0000000..ce1d9e4 --- /dev/null +++ b/modules/binarylane/main.tf @@ -0,0 +1,248 @@ +locals { + # home WAN address of prodnxsr0020, which terminates one wg link per vpn host + router_endpoint = "103.216.191.185" + wg_port = 51820 + domain = "main.unkin.net" + + # lab destinations reached through each region's vpn host; SMTP stays on the public internet + lab_routes = [ + "198.18.0.0/16", + "10.10.12.200/32", # management ssh host + ] + + # vpc route entries must not overlap the vpcs themselves: the /16 minus 198.18.30.0/23 + vpc_lab_routes = concat([ + "198.18.0.0/20", + "198.18.16.0/21", + "198.18.24.0/22", + "198.18.28.0/23", + "198.18.32.0/19", + "198.18.64.0/18", + "198.18.128.0/17", + ], [for dst in local.lab_routes : dst if dst != "198.18.0.0/16"]) + + # ospf hellos over the p2p links; wg has one peer per interface so this routes to it + ospf_allspf = "224.0.0.5/32" + + vpcs = { + syd = "198.18.30.0/24" + mel = "198.18.31.0/24" + } + + vpns = { + ausyd2nxvm2131 = { region = "syd", vpc_ip = "198.18.30.11", router_port = 51821, router_address = "198.18.2.176", wg_address = "198.18.2.177", loopback = "198.18.2.161" } + aumel1nxvm2133 = { region = "mel", vpc_ip = "198.18.31.11", router_port = 51822, router_address = "198.18.2.178", wg_address = "198.18.2.179", loopback = "198.18.2.162" } + } + + edges = { + ausyd2nxvm2130 = { region = "syd", vpc_ip = "198.18.30.10", vpn = "ausyd2nxvm2131", ptr = "mta1.edge.unkin.net" } + aumel1nxvm2132 = { region = "mel", vpc_ip = "198.18.31.10", vpn = "aumel1nxvm2133", ptr = "mta2.edge.unkin.net" } + } +} + +resource "binarylane_vpc" "dmz" { + for_each = local.vpcs + + name = "dmz-${each.key}" + ip_range = each.value +} + +resource "binarylane_vpc_route_entries" "dmz" { + for_each = local.vpns + + vpc_id = binarylane_vpc.dmz[each.value.region].id + route_entries = [for dst in local.vpc_lab_routes : { + description = "lab via ${each.key}" + destination = dst + router = binarylane_server.vpn[each.key].private_ipv4_addresses[0] + }] +} + +resource "wireguard_asymmetric_key" "router" { + for_each = local.vpns +} + +resource "wireguard_asymmetric_key" "vpn" { + for_each = local.vpns +} + +resource "vault_kv_secret_v2" "router" { + for_each = local.vpns + + mount = "kv" + name = "service/wireguard/binarylane/prodnxsr0020-${each.value.region}" + data_json = jsonencode({ + private_key = wireguard_asymmetric_key.router[each.key].private_key + public_key = wireguard_asymmetric_key.router[each.key].public_key + address = "${each.value.router_address}/31" + listen_port = each.value.router_port + }) +} + +resource "vault_kv_secret_v2" "vpn" { + for_each = local.vpns + + mount = "kv" + name = "service/wireguard/binarylane/${each.key}" + data_json = jsonencode({ + private_key = wireguard_asymmetric_key.vpn[each.key].private_key + public_key = wireguard_asymmetric_key.vpn[each.key].public_key + address = "${each.value.wg_address}/31" + listen_port = local.wg_port + }) +} + +resource "binarylane_server" "vpn" { + for_each = local.vpns + + name = "${each.key}.${local.domain}" + region = each.value.region + image = "alma-9" + size = "std-min" + public_ipv4_count = 1 + vpc_id = binarylane_vpc.dmz[each.value.region].id + vpc_ipv4_address = each.value.vpc_ip + source_and_destination_check = false + backups = false + ipv6 = false + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + hostname = each.key + fqdn = "${each.key}.${local.domain}" + vpn = { + address = "${each.value.wg_address}/31" + loopback = "${each.value.loopback}/32" + routes = local.lab_routes + src = each.value.wg_address + port = local.wg_port + private_key = wireguard_asymmetric_key.vpn[each.key].private_key + router_key = wireguard_asymmetric_key.router[each.key].public_key + router = "${local.router_endpoint}:${each.value.router_port}" + allowed_ips = join(", ", concat(local.lab_routes, [local.ospf_allspf])) + } + routes = null + }) + + # user_data/image changes reinstall the server in place, wiping its disk + lifecycle { + ignore_changes = [user_data, image] + prevent_destroy = true + } +} + +resource "binarylane_server" "edge" { + for_each = local.edges + + name = "${each.key}.${local.domain}" + region = each.value.region + image = "alma-9" + size = "std-min" + public_ipv4_count = 1 + vpc_id = binarylane_vpc.dmz[each.value.region].id + vpc_ipv4_address = each.value.vpc_ip + port_blocking = false + backups = false + ipv6 = false + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + hostname = each.key + fqdn = "${each.key}.${local.domain}" + vpn = null + routes = { + vpc = local.vpcs[each.value.region] + list = join(", ", [ + for dst in local.lab_routes : "${dst} ${local.vpns[each.value.vpn].vpc_ip} src=${each.value.vpc_ip}" + ]) + } + }) + + # user_data/image changes reinstall the server in place, wiping its disk + lifecycle { + ignore_changes = [user_data, image] + prevent_destroy = true + } +} + +resource "binarylane_server_reverse_name" "edge" { + for_each = local.edges + + server_id = binarylane_server.edge[each.key].id + ipv4_address = binarylane_server.edge[each.key].public_ipv4_addresses[0] + reverse_name = each.value.ptr +} + +resource "binarylane_server_firewall_rules" "edge" { + for_each = local.edges + + server_id = binarylane_server.edge[each.key].id + firewall_rules = [ + { + description = "smtp" + protocol = "tcp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + destination_ports = ["25"] + action = "accept" + }, + { + description = "ssh from the lab via ${each.value.vpn}" + protocol = "tcp" + source_addresses = ["${local.vpns[each.value.vpn].vpc_ip}/32", "10.10.12.200/32"] + destination_addresses = [each.value.vpc_ip] + destination_ports = ["22"] + action = "accept" + }, + { + description = "icmp" + protocol = "icmp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "accept" + }, + { + description = "default deny" + protocol = "all" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "drop" + }, + ] +} + +# SSH has no rule: management reaches sshd through wg0, which this firewall never sees. +resource "binarylane_server_firewall_rules" "vpn" { + for_each = local.vpns + + server_id = binarylane_server.vpn[each.key].id + firewall_rules = [ + { + description = "wireguard from prodnxsr0020" + protocol = "udp" + source_addresses = ["${local.router_endpoint}/32"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + destination_ports = [tostring(local.wg_port)] + action = "accept" + }, + { + description = "forward dmz to lab" + protocol = "all" + source_addresses = [local.vpcs[each.value.region]] + destination_addresses = local.lab_routes + action = "accept" + }, + { + description = "icmp" + protocol = "icmp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "accept" + }, + { + description = "default deny" + protocol = "all" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "drop" + }, + ] +} diff --git a/modules/binarylane/outputs.tf b/modules/binarylane/outputs.tf new file mode 100644 index 0000000..0bf347d --- /dev/null +++ b/modules/binarylane/outputs.tf @@ -0,0 +1,35 @@ +output "hosts" { + description = "Hostname, public IPv4, VPC IPv4 and (vpn hosts) wg address, loopback and public key per host." + value = merge( + { for k, v in binarylane_server.edge : k => { + hostname = v.name + public_ipv4 = v.public_ipv4_addresses[0] + vpc_ipv4 = local.edges[k].vpc_ip + } }, + { for k, v in binarylane_server.vpn : k => { + hostname = v.name + public_ipv4 = v.public_ipv4_addresses[0] + vpc_ipv4 = local.vpns[k].vpc_ip + wg_address = local.vpns[k].wg_address + loopback = local.vpns[k].loopback + wg_pubkey = wireguard_asymmetric_key.vpn[k].public_key + } }, + ) +} + +output "router_links" { + description = "Per-region p2p wg/ospf link as prodnxsr0020 sees it; vpc_cidr arrives via ospf. Consumed by its Puppet config." + value = { for k, v in local.vpns : v.region => { + interface = "wg-${v.region}" + listen_port = v.router_port + router_address = "${v.router_address}/31" + peer_address = "${v.wg_address}/31" + peer_loopback = "${v.loopback}/32" + allowed_ips = ["${v.wg_address}/32", "${v.loopback}/32", local.vpcs[v.region], local.ospf_allspf] + peer_public_key = wireguard_asymmetric_key.vpn[k].public_key + peer_endpoint = "${binarylane_server.vpn[k].public_ipv4_addresses[0]}:${local.wg_port}" + vpc_cidr = local.vpcs[v.region] + private_key_vault_path = "kv/${vault_kv_secret_v2.router[k].name}" + peer_key_vault_path = "kv/${vault_kv_secret_v2.vpn[k].name}" + } } +} diff --git a/modules/binarylane/providers.tf b/modules/binarylane/providers.tf new file mode 100644 index 0000000..68e53cf --- /dev/null +++ b/modules/binarylane/providers.tf @@ -0,0 +1,8 @@ +provider "binarylane" { + api_token = var.binarylane_token +} + +# VAULT_ADDR / VAULT_TOKEN come from the Makefile; the k8s auth role token cannot mint child tokens. +provider "vault" { + skip_child_token = true +} diff --git a/modules/binarylane/versions.tf b/modules/binarylane/versions.tf index ff7164f..e323c20 100644 --- a/modules/binarylane/versions.tf +++ b/modules/binarylane/versions.tf @@ -1,3 +1,18 @@ terraform { required_version = ">= 1.10" + + required_providers { + binarylane = { + source = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" + version = "0.14.0" + } + vault = { + source = "hashicorp/vault" + version = "5.6.0" + } + wireguard = { + source = "OJFord/wireguard" + version = "0.4.1" + } + } }