From 0ee9dac5b0a576f01b86e79ef53d8f222501d049 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 21:50:09 +1100 Subject: [PATCH 1/9] Add mta1/mta2 DMZ mail gateways with WireGuard bootstrap --- environments/global/.terraform.lock.hcl | 50 ++++++++ modules/binarylane/.terraform.lock.hcl | 50 ++++++++ modules/binarylane/.tflint.hcl | 3 - modules/binarylane/cloud-init.yaml.tftpl | 37 ++++++ modules/binarylane/main.tf | 142 +++++++++++++++++++++++ modules/binarylane/outputs.tf | 15 +++ modules/binarylane/providers.tf | 8 ++ modules/binarylane/versions.tf | 15 +++ 8 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 environments/global/.terraform.lock.hcl create mode 100644 modules/binarylane/.terraform.lock.hcl delete mode 100644 modules/binarylane/.tflint.hcl create mode 100644 modules/binarylane/cloud-init.yaml.tftpl create mode 100644 modules/binarylane/main.tf create mode 100644 modules/binarylane/outputs.tf create mode 100644 modules/binarylane/providers.tf diff --git a/environments/global/.terraform.lock.hcl b/environments/global/.terraform.lock.hcl new file mode 100644 index 0000000..39ba811 --- /dev/null +++ b/environments/global/.terraform.lock.hcl @@ -0,0 +1,50 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" { + version = "0.14.0" + constraints = "0.14.0" + hashes = [ + "h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=", + "zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45", + ] +} + +provider "registry.opentofu.org/hashicorp/vault" { + version = "5.6.0" + constraints = "5.6.0" + hashes = [ + "h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=", + "zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8", + "zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a", + "zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8", + "zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7", + "zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf", + "zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95", + "zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d", + "zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21", + "zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b", + ] +} + +provider "registry.opentofu.org/ojford/wireguard" { + version = "0.4.1" + constraints = "0.4.1" + hashes = [ + "h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620", + "zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1", + "zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46", + "zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7", + "zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2", + "zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2", + "zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92", + "zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1", + "zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863", + "zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b", + "zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e", + "zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37", + "zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1", + ] +} diff --git a/modules/binarylane/.terraform.lock.hcl b/modules/binarylane/.terraform.lock.hcl new file mode 100644 index 0000000..39ba811 --- /dev/null +++ b/modules/binarylane/.terraform.lock.hcl @@ -0,0 +1,50 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" { + version = "0.14.0" + constraints = "0.14.0" + hashes = [ + "h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=", + "zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45", + ] +} + +provider "registry.opentofu.org/hashicorp/vault" { + version = "5.6.0" + constraints = "5.6.0" + hashes = [ + "h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=", + "zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8", + "zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a", + "zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8", + "zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7", + "zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf", + "zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95", + "zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d", + "zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21", + "zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b", + ] +} + +provider "registry.opentofu.org/ojford/wireguard" { + version = "0.4.1" + constraints = "0.4.1" + hashes = [ + "h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620", + "zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1", + "zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46", + "zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7", + "zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2", + "zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2", + "zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92", + "zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1", + "zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863", + "zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b", + "zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e", + "zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37", + "zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1", + ] +} diff --git a/modules/binarylane/.tflint.hcl b/modules/binarylane/.tflint.hcl deleted file mode 100644 index 3a2ace1..0000000 --- a/modules/binarylane/.tflint.hcl +++ /dev/null @@ -1,3 +0,0 @@ -rule "terraform_unused_declarations" { - enabled = false -} diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl new file mode 100644 index 0000000..78cbd88 --- /dev/null +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -0,0 +1,37 @@ +#cloud-config +hostname: ${hostname} +fqdn: ${fqdn} +prefer_fqdn_over_hostname: true + +write_files: + - path: /etc/wireguard/wg0.conf + permissions: "0600" + content: | + [Interface] + Address = ${wg_address}/24 + ListenPort = ${wg_port} + PrivateKey = ${wg_private_key} + + [Peer] + PublicKey = ${hub_public_key} + Endpoint = ${hub_endpoint} + AllowedIPs = ${allowed_ips} + PersistentKeepalive = 25 + - path: /etc/NetworkManager/conf.d/90-dns-none.conf + content: | + [main] + dns=none + +runcmd: + - dnf install -y wireguard-tools + - systemctl enable --now wg-quick@wg0 + - systemctl reload NetworkManager + - printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf + - rpm -q puppet-agent || dnf install -y https://yum.puppet.com/puppet8-release-el-9.noarch.rpm + - dnf install -y puppet-agent + - /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn} + - /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net + - /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net + - /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net + - /opt/puppetlabs/bin/puppet config set --section main environment production + - systemctl enable --now puppet diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf new file mode 100644 index 0000000..01562e3 --- /dev/null +++ b/modules/binarylane/main.tf @@ -0,0 +1,142 @@ +locals { + # home gateway (prodnxsr0020) end of the mail DMZ tunnel + hub_endpoint = "103.216.191.185" + hub_port = 51821 + hub_address = "10.10.13.1" + wg_port = 51820 + + # only these destinations route over wg0; everything else (incl. SMTP) stays public + wg_allowed_ips = [ + "10.10.13.1/32", # prodnxsr0020 tunnel address + "10.10.12.200/32", # management ssh client (wireguard workstation) + "198.18.200.2/32", # puppetca.k8s.syd1.au.unkin.net + "198.18.200.3/32", # puppet.k8s.syd1.au.unkin.net + "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net (package repos) + "198.18.200.7/32", # bind-internal resolvers + "198.18.200.17/32", # rspamd milter :11332 + "198.18.25.78/32", # vault.service.consul + "198.18.26.197/32", # vault.service.consul + "198.18.27.116/32", # vault.service.consul + "198.18.28.248/32", # vault.service.consul + "198.18.29.92/32", # vault.service.consul + ] + + vpcs = { + syd = "10.10.14.0/24" + mel = "10.10.15.0/24" + } + + gateways = { + mta1 = { region = "syd", wg_address = "10.10.13.11" } + mta2 = { region = "mel", wg_address = "10.10.13.12" } + } +} + +resource "binarylane_vpc" "dmz" { + for_each = local.vpcs + + name = "dmz-${each.key}" + ip_range = each.value +} + +resource "wireguard_asymmetric_key" "hub" {} + +resource "wireguard_asymmetric_key" "gateway" { + for_each = local.gateways +} + +resource "vault_kv_secret_v2" "hub" { + mount = "kv" + name = "service/wireguard/binarylane/prodnxsr0020" + data_json = jsonencode({ + private_key = wireguard_asymmetric_key.hub.private_key + public_key = wireguard_asymmetric_key.hub.public_key + address = "${local.hub_address}/24" + listen_port = local.hub_port + }) +} + +resource "vault_kv_secret_v2" "gateway" { + for_each = local.gateways + + mount = "kv" + name = "service/wireguard/binarylane/${each.key}" + data_json = jsonencode({ + private_key = wireguard_asymmetric_key.gateway[each.key].private_key + public_key = wireguard_asymmetric_key.gateway[each.key].public_key + address = "${each.value.wg_address}/24" + listen_port = local.wg_port + }) +} + +resource "binarylane_server" "gateway" { + for_each = local.gateways + + name = "${each.key}.edge.unkin.net" + region = each.value.region + image = "alma-9" + size = "std-min" + public_ipv4_count = 1 + vpc_id = binarylane_vpc.dmz[each.value.region].id + port_blocking = false + backups = false + ipv6 = false + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + hostname = each.key + fqdn = "${each.key}.edge.unkin.net" + wg_address = each.value.wg_address + wg_port = local.wg_port + wg_private_key = wireguard_asymmetric_key.gateway[each.key].private_key + hub_public_key = wireguard_asymmetric_key.hub.public_key + hub_endpoint = "${local.hub_endpoint}:${local.hub_port}" + allowed_ips = join(", ", local.wg_allowed_ips) + }) +} + +resource "binarylane_server_reverse_name" "gateway" { + for_each = local.gateways + + server_id = binarylane_server.gateway[each.key].id + ipv4_address = binarylane_server.gateway[each.key].public_ipv4_addresses[0] + reverse_name = "${each.key}.edge.unkin.net" +} + +# SSH has no public rule: management reaches sshd through wg0, which this firewall never sees. +resource "binarylane_server_firewall_rules" "gateway" { + for_each = local.gateways + + server_id = binarylane_server.gateway[each.key].id + firewall_rules = [ + { + description = "mail and https" + protocol = "tcp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + destination_ports = ["25", "465", "587", "993", "443"] + action = "accept" + }, + { + description = "wireguard from home gateway" + protocol = "udp" + source_addresses = ["${local.hub_endpoint}/32"] + destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + destination_ports = [tostring(local.wg_port)] + action = "accept" + }, + { + description = "icmp" + protocol = "icmp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + action = "accept" + }, + { + description = "default deny" + protocol = "all" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + action = "drop" + }, + ] +} diff --git a/modules/binarylane/outputs.tf b/modules/binarylane/outputs.tf new file mode 100644 index 0000000..d2ea70c --- /dev/null +++ b/modules/binarylane/outputs.tf @@ -0,0 +1,15 @@ +output "public_ipv4" { + description = "Public IPv4 per gateway." + value = { for k, v in binarylane_server.gateway : k => v.public_ipv4_addresses[0] } +} + +output "wireguard" { + description = "Tunnel address and WireGuard public key per gateway, plus the prodnxsr0020 end." + value = { + hub = { address = local.hub_address, public_key = wireguard_asymmetric_key.hub.public_key, listen_port = local.hub_port } + gateways = { for k, v in local.gateways : k => { + address = v.wg_address + public_key = wireguard_asymmetric_key.gateway[k].public_key + } } + } +} diff --git a/modules/binarylane/providers.tf b/modules/binarylane/providers.tf new file mode 100644 index 0000000..68e53cf --- /dev/null +++ b/modules/binarylane/providers.tf @@ -0,0 +1,8 @@ +provider "binarylane" { + api_token = var.binarylane_token +} + +# VAULT_ADDR / VAULT_TOKEN come from the Makefile; the k8s auth role token cannot mint child tokens. +provider "vault" { + skip_child_token = true +} diff --git a/modules/binarylane/versions.tf b/modules/binarylane/versions.tf index ff7164f..e323c20 100644 --- a/modules/binarylane/versions.tf +++ b/modules/binarylane/versions.tf @@ -1,3 +1,18 @@ terraform { required_version = ">= 1.10" + + required_providers { + binarylane = { + source = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" + version = "0.14.0" + } + vault = { + source = "hashicorp/vault" + version = "5.6.0" + } + wireguard = { + source = "OJFord/wireguard" + version = "0.4.1" + } + } } From fe6dc0bc95de06b7d758a769eb4854ed4083d621 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 21:52:06 +1100 Subject: [PATCH 2/9] Align comments to tofu 1.10 fmt --- modules/binarylane/main.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 01562e3..02162c7 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -7,18 +7,18 @@ locals { # only these destinations route over wg0; everything else (incl. SMTP) stays public wg_allowed_ips = [ - "10.10.13.1/32", # prodnxsr0020 tunnel address - "10.10.12.200/32", # management ssh client (wireguard workstation) - "198.18.200.2/32", # puppetca.k8s.syd1.au.unkin.net - "198.18.200.3/32", # puppet.k8s.syd1.au.unkin.net - "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net (package repos) - "198.18.200.7/32", # bind-internal resolvers - "198.18.200.17/32", # rspamd milter :11332 - "198.18.25.78/32", # vault.service.consul - "198.18.26.197/32", # vault.service.consul - "198.18.27.116/32", # vault.service.consul - "198.18.28.248/32", # vault.service.consul - "198.18.29.92/32", # vault.service.consul + "10.10.13.1/32", # prodnxsr0020 tunnel address + "10.10.12.200/32", # management ssh client (wireguard workstation) + "198.18.200.2/32", # puppetca.k8s.syd1.au.unkin.net + "198.18.200.3/32", # puppet.k8s.syd1.au.unkin.net + "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net (package repos) + "198.18.200.7/32", # bind-internal resolvers + "198.18.200.17/32", # rspamd milter :11332 + "198.18.25.78/32", # vault.service.consul + "198.18.26.197/32", # vault.service.consul + "198.18.27.116/32", # vault.service.consul + "198.18.28.248/32", # vault.service.consul + "198.18.29.92/32", # vault.service.consul ] vpcs = { From 975ff15b6bb498f57eaa4a9f6b4a56915d8e0594 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 21:57:40 +1100 Subject: [PATCH 3/9] Bootstrap gateways via a retrying wg handshake unit and guard servers against rebuilds --- README.md | 6 +++ environments/global/.terraform.lock.hcl | 50 ------------------------ modules/binarylane/cloud-init.yaml.tftpl | 37 ++++++++++++++---- modules/binarylane/main.tf | 6 +++ 4 files changed, 41 insertions(+), 58 deletions(-) delete mode 100644 environments/global/.terraform.lock.hcl diff --git a/README.md b/README.md index b2069fb..63966c6 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,9 @@ The BinaryLane API token is read from `kv/service/terraform/binarylane` (field from `consul_root/au/syd1/creds/terraform-binarylane`. Both are granted to the `terraform_binarylane` AppRole and the `woodpecker_terraform_binarylane` k8s role in terraform-vault. + +## Gateways + +`mta1`/`mta2` bootstrap wg0 from a key in `user_data` (readable via the +BinaryLane API); Puppet then manages wg0 from Vault, so rotate by replacing +`wireguard_asymmetric_key.gateway` here and letting Puppet roll it out. diff --git a/environments/global/.terraform.lock.hcl b/environments/global/.terraform.lock.hcl deleted file mode 100644 index 39ba811..0000000 --- a/environments/global/.terraform.lock.hcl +++ /dev/null @@ -1,50 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" { - version = "0.14.0" - constraints = "0.14.0" - hashes = [ - "h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=", - "zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45", - ] -} - -provider "registry.opentofu.org/hashicorp/vault" { - version = "5.6.0" - constraints = "5.6.0" - hashes = [ - "h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=", - "zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8", - "zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a", - "zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8", - "zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7", - "zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf", - "zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95", - "zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d", - "zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21", - "zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b", - ] -} - -provider "registry.opentofu.org/ojford/wireguard" { - version = "0.4.1" - constraints = "0.4.1" - hashes = [ - "h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620", - "zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1", - "zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46", - "zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7", - "zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2", - "zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2", - "zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92", - "zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1", - "zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863", - "zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b", - "zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e", - "zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37", - "zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1", - ] -} diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl index 78cbd88..c362ed2 100644 --- a/modules/binarylane/cloud-init.yaml.tftpl +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -17,21 +17,42 @@ write_files: Endpoint = ${hub_endpoint} AllowedIPs = ${allowed_ips} PersistentKeepalive = 25 - - path: /etc/NetworkManager/conf.d/90-dns-none.conf + - path: /usr/local/sbin/wg-bootstrap + permissions: "0755" content: | - [main] - dns=none + #!/bin/sh + set -e + wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }' + printf '[main]\ndns=none\n' > /etc/NetworkManager/conf.d/90-dns-none.conf + systemctl reload NetworkManager + printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf + systemctl enable --now puppet + systemctl disable wg-bootstrap + - path: /etc/systemd/system/wg-bootstrap.service + content: | + [Unit] + Description=Switch to internal DNS and start puppet once wg0 has a handshake + After=wg-quick@wg0.service + StartLimitIntervalSec=0 + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/wg-bootstrap + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target runcmd: - dnf install -y wireguard-tools - systemctl enable --now wg-quick@wg0 - - systemctl reload NetworkManager - - printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf - - rpm -q puppet-agent || dnf install -y https://yum.puppet.com/puppet8-release-el-9.noarch.rpm - - dnf install -y puppet-agent + - rpm -q openvox-agent || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm + - dnf install -y openvox-agent - /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn} - /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net - /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net - /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net - /opt/puppetlabs/bin/puppet config set --section main environment production - - systemctl enable --now puppet + - systemctl daemon-reload + - systemctl enable --now --no-block wg-bootstrap diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 02162c7..1ac1bce 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -92,6 +92,12 @@ resource "binarylane_server" "gateway" { hub_endpoint = "${local.hub_endpoint}:${local.hub_port}" allowed_ips = join(", ", local.wg_allowed_ips) }) + + # user_data/image changes reinstall the server in place, wiping its disk + lifecycle { + ignore_changes = [user_data, image] + prevent_destroy = true + } } resource "binarylane_server_reverse_name" "gateway" { From d4d62ffcf42738298861fe0652b51c8ed52c902d Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 22:08:09 +1100 Subject: [PATCH 4/9] Split each DMZ into a postfix edge host and a WireGuard vpn host --- README.md | 11 +- modules/binarylane/cloud-init.yaml.tftpl | 71 +++++--- modules/binarylane/main.tf | 196 +++++++++++++++++------ modules/binarylane/outputs.tf | 31 ++-- 4 files changed, 215 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 63966c6..f77953c 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,11 @@ from `consul_root/au/syd1/creds/terraform-binarylane`. Both are granted to the `terraform_binarylane` AppRole and the `woodpecker_terraform_binarylane` k8s role in terraform-vault. -## Gateways +## Mail edge -`mta1`/`mta2` bootstrap wg0 from a key in `user_data` (readable via the -BinaryLane API); Puppet then manages wg0 from Vault, so rotate by replacing -`wireguard_asymmetric_key.gateway` here and letting Puppet roll it out. +Each DMZ VPC (`syd`, `mel`) holds a postfix edge host and a vpn host. The vpn +host runs wg0 to prodnxsr0020 and forwards the lab routes for its VPC; the edge +host reaches the lab through it and has no WireGuard. wg0 is bootstrapped from +a key in `user_data` (readable via the BinaryLane API); Puppet then manages it +from Vault, so rotate by replacing `wireguard_asymmetric_key.vpn` here and +letting Puppet roll it out. diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl index c362ed2..15cac91 100644 --- a/modules/binarylane/cloud-init.yaml.tftpl +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -4,40 +4,70 @@ fqdn: ${fqdn} prefer_fqdn_over_hostname: true write_files: +%{ if vpn != null ~} - path: /etc/wireguard/wg0.conf permissions: "0600" content: | [Interface] - Address = ${wg_address}/24 - ListenPort = ${wg_port} - PrivateKey = ${wg_private_key} + Address = ${vpn.address}/24 + ListenPort = ${vpn.port} + PrivateKey = ${vpn.private_key} [Peer] - PublicKey = ${hub_public_key} - Endpoint = ${hub_endpoint} - AllowedIPs = ${allowed_ips} + PublicKey = ${vpn.hub_key} + Endpoint = ${vpn.hub} + AllowedIPs = ${vpn.allowed_ips} PersistentKeepalive = 25 - - path: /usr/local/sbin/wg-bootstrap + - path: /etc/sysctl.d/90-ip-forward.conf + content: | + net.ipv4.ip_forward = 1 +%{ endif ~} + - path: /usr/local/sbin/lab-bootstrap permissions: "0755" content: | - #!/bin/sh - set -e + #!/bin/bash + set -euo pipefail + if ! rpm -q openvox-agent >/dev/null; then + rpm -q openvox7-release >/dev/null || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm + dnf install -y openvox-agent + fi + /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn} + /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net + /opt/puppetlabs/bin/puppet config set --section main environment production +%{ if vpn != null ~} + rpm -q wireguard-tools >/dev/null || dnf install -y wireguard-tools + sysctl -p /etc/sysctl.d/90-ip-forward.conf + systemctl enable --now wg-quick@wg0 wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }' +%{ endif ~} +%{ if routes != null ~} + if ! ip route show 198.18.200.3 | grep -q .; then + dev=$(ip -o -4 addr show to ${routes.vpc} | awk '{ print $2; exit }') + con=$(nmcli -g GENERAL.CONNECTION device show "$dev") + nmcli connection modify "$con" ipv4.routes "${routes.list}" + nmcli device reapply "$dev" + fi +%{ endif ~} + timeout 5 bash -c ' /etc/NetworkManager/conf.d/90-dns-none.conf - systemctl reload NetworkManager + nmcli general reload + rm -f /etc/resolv.conf printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf systemctl enable --now puppet - systemctl disable wg-bootstrap - - path: /etc/systemd/system/wg-bootstrap.service + systemctl disable lab-bootstrap + - path: /etc/systemd/system/lab-bootstrap.service content: | [Unit] - Description=Switch to internal DNS and start puppet once wg0 has a handshake - After=wg-quick@wg0.service + Description=Install openvox, wait for the lab via the vpn host, then switch to internal DNS and start puppet + Wants=network-online.target + After=network-online.target StartLimitIntervalSec=0 [Service] Type=oneshot - ExecStart=/usr/local/sbin/wg-bootstrap + ExecStart=/usr/local/sbin/lab-bootstrap Restart=on-failure RestartSec=30 @@ -45,14 +75,5 @@ write_files: WantedBy=multi-user.target runcmd: - - dnf install -y wireguard-tools - - systemctl enable --now wg-quick@wg0 - - rpm -q openvox-agent || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm - - dnf install -y openvox-agent - - /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn} - - /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net - - /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net - - /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net - - /opt/puppetlabs/bin/puppet config set --section main environment production - systemctl daemon-reload - - systemctl enable --now --no-block wg-bootstrap + - systemctl enable --now --no-block lab-bootstrap diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 1ac1bce..850fa8c 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -4,16 +4,16 @@ locals { hub_port = 51821 hub_address = "10.10.13.1" wg_port = 51820 + domain = "main.unkin.net" - # only these destinations route over wg0; everything else (incl. SMTP) stays public - wg_allowed_ips = [ - "10.10.13.1/32", # prodnxsr0020 tunnel address - "10.10.12.200/32", # management ssh client (wireguard workstation) + # lab destinations reached through each region's vpn host; SMTP stays on the public internet + lab_routes = [ + "10.10.12.200/32", # management ssh host + "198.18.199.0/32", # git.unkin.net (rspamd rules) "198.18.200.2/32", # puppetca.k8s.syd1.au.unkin.net "198.18.200.3/32", # puppet.k8s.syd1.au.unkin.net - "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net (package repos) + "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net "198.18.200.7/32", # bind-internal resolvers - "198.18.200.17/32", # rspamd milter :11332 "198.18.25.78/32", # vault.service.consul "198.18.26.197/32", # vault.service.consul "198.18.27.116/32", # vault.service.consul @@ -26,9 +26,14 @@ locals { mel = "10.10.15.0/24" } - gateways = { - mta1 = { region = "syd", wg_address = "10.10.13.11" } - mta2 = { region = "mel", wg_address = "10.10.13.12" } + vpns = { + ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", wg_address = "10.10.13.11" } + aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", wg_address = "10.10.13.12" } + } + + edges = { + ausyd2nxvm2130 = { region = "syd", vpc_ip = "10.10.14.10", vpn = "ausyd2nxvm2131", ptr = "mta1.edge.unkin.net" } + aumel1nxvm2132 = { region = "mel", vpc_ip = "10.10.15.10", vpn = "aumel1nxvm2133", ptr = "mta2.edge.unkin.net" } } } @@ -39,10 +44,21 @@ resource "binarylane_vpc" "dmz" { ip_range = each.value } +resource "binarylane_vpc_route_entries" "dmz" { + for_each = local.vpns + + vpc_id = binarylane_vpc.dmz[each.value.region].id + route_entries = [for dst in local.lab_routes : { + description = "lab via ${each.key}" + destination = dst + router = binarylane_server.vpn[each.key].private_ipv4_addresses[0] + }] +} + resource "wireguard_asymmetric_key" "hub" {} -resource "wireguard_asymmetric_key" "gateway" { - for_each = local.gateways +resource "wireguard_asymmetric_key" "vpn" { + for_each = local.vpns } resource "vault_kv_secret_v2" "hub" { @@ -56,41 +72,45 @@ resource "vault_kv_secret_v2" "hub" { }) } -resource "vault_kv_secret_v2" "gateway" { - for_each = local.gateways +resource "vault_kv_secret_v2" "vpn" { + for_each = local.vpns mount = "kv" name = "service/wireguard/binarylane/${each.key}" data_json = jsonencode({ - private_key = wireguard_asymmetric_key.gateway[each.key].private_key - public_key = wireguard_asymmetric_key.gateway[each.key].public_key + private_key = wireguard_asymmetric_key.vpn[each.key].private_key + public_key = wireguard_asymmetric_key.vpn[each.key].public_key address = "${each.value.wg_address}/24" listen_port = local.wg_port }) } -resource "binarylane_server" "gateway" { - for_each = local.gateways +resource "binarylane_server" "vpn" { + for_each = local.vpns - name = "${each.key}.edge.unkin.net" - region = each.value.region - image = "alma-9" - size = "std-min" - public_ipv4_count = 1 - vpc_id = binarylane_vpc.dmz[each.value.region].id - port_blocking = false - backups = false - ipv6 = false + name = "${each.key}.${local.domain}" + region = each.value.region + image = "alma-9" + size = "std-min" + public_ipv4_count = 1 + vpc_id = binarylane_vpc.dmz[each.value.region].id + vpc_ipv4_address = each.value.vpc_ip + source_and_destination_check = false + backups = false + ipv6 = false user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { - hostname = each.key - fqdn = "${each.key}.edge.unkin.net" - wg_address = each.value.wg_address - wg_port = local.wg_port - wg_private_key = wireguard_asymmetric_key.gateway[each.key].private_key - hub_public_key = wireguard_asymmetric_key.hub.public_key - hub_endpoint = "${local.hub_endpoint}:${local.hub_port}" - allowed_ips = join(", ", local.wg_allowed_ips) + hostname = each.key + fqdn = "${each.key}.${local.domain}" + vpn = { + address = each.value.wg_address + port = local.wg_port + private_key = wireguard_asymmetric_key.vpn[each.key].private_key + hub_key = wireguard_asymmetric_key.hub.public_key + hub = "${local.hub_endpoint}:${local.hub_port}" + allowed_ips = join(", ", concat(["${local.hub_address}/32"], local.lab_routes)) + } + routes = null }) # user_data/image changes reinstall the server in place, wiping its disk @@ -100,48 +120,118 @@ resource "binarylane_server" "gateway" { } } -resource "binarylane_server_reverse_name" "gateway" { - for_each = local.gateways +resource "binarylane_server" "edge" { + for_each = local.edges - server_id = binarylane_server.gateway[each.key].id - ipv4_address = binarylane_server.gateway[each.key].public_ipv4_addresses[0] - reverse_name = "${each.key}.edge.unkin.net" + name = "${each.key}.${local.domain}" + region = each.value.region + image = "alma-9" + size = "std-min" + public_ipv4_count = 1 + vpc_id = binarylane_vpc.dmz[each.value.region].id + vpc_ipv4_address = each.value.vpc_ip + port_blocking = false + backups = false + ipv6 = false + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + hostname = each.key + fqdn = "${each.key}.${local.domain}" + vpn = null + routes = { + vpc = local.vpcs[each.value.region] + list = join(", ", [ + for dst in local.lab_routes : "${dst} ${local.vpns[each.value.vpn].vpc_ip} src=${each.value.vpc_ip}" + ]) + } + }) + + # user_data/image changes reinstall the server in place, wiping its disk + lifecycle { + ignore_changes = [user_data, image] + prevent_destroy = true + } } -# SSH has no public rule: management reaches sshd through wg0, which this firewall never sees. -resource "binarylane_server_firewall_rules" "gateway" { - for_each = local.gateways +resource "binarylane_server_reverse_name" "edge" { + for_each = local.edges - server_id = binarylane_server.gateway[each.key].id + server_id = binarylane_server.edge[each.key].id + ipv4_address = binarylane_server.edge[each.key].public_ipv4_addresses[0] + reverse_name = each.value.ptr +} + +resource "binarylane_server_firewall_rules" "edge" { + for_each = local.edges + + server_id = binarylane_server.edge[each.key].id firewall_rules = [ { - description = "mail and https" + description = "smtp" protocol = "tcp" source_addresses = ["0.0.0.0/0"] - destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] - destination_ports = ["25", "465", "587", "993", "443"] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + destination_ports = ["25"] action = "accept" }, { - description = "wireguard from home gateway" - protocol = "udp" - source_addresses = ["${local.hub_endpoint}/32"] - destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] - destination_ports = [tostring(local.wg_port)] + description = "ssh from the lab via ${each.value.vpn}" + protocol = "tcp" + source_addresses = ["${local.vpns[each.value.vpn].vpc_ip}/32", "10.10.12.200/32"] + destination_addresses = [each.value.vpc_ip] + destination_ports = ["22"] action = "accept" }, { description = "icmp" protocol = "icmp" source_addresses = ["0.0.0.0/0"] - destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] action = "accept" }, { description = "default deny" protocol = "all" source_addresses = ["0.0.0.0/0"] - destination_addresses = [binarylane_server.gateway[each.key].public_ipv4_addresses[0]] + destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "drop" + }, + ] +} + +# SSH has no rule: management reaches sshd through wg0, which this firewall never sees. +resource "binarylane_server_firewall_rules" "vpn" { + for_each = local.vpns + + server_id = binarylane_server.vpn[each.key].id + firewall_rules = [ + { + description = "wireguard from home gateway" + protocol = "udp" + source_addresses = ["${local.hub_endpoint}/32"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + destination_ports = [tostring(local.wg_port)] + action = "accept" + }, + { + description = "forward dmz to lab" + protocol = "all" + source_addresses = [local.vpcs[each.value.region]] + destination_addresses = local.lab_routes + action = "accept" + }, + { + description = "icmp" + protocol = "icmp" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] + action = "accept" + }, + { + description = "default deny" + protocol = "all" + source_addresses = ["0.0.0.0/0"] + destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] action = "drop" }, ] diff --git a/modules/binarylane/outputs.tf b/modules/binarylane/outputs.tf index d2ea70c..a45f9ea 100644 --- a/modules/binarylane/outputs.tf +++ b/modules/binarylane/outputs.tf @@ -1,15 +1,22 @@ -output "public_ipv4" { - description = "Public IPv4 per gateway." - value = { for k, v in binarylane_server.gateway : k => v.public_ipv4_addresses[0] } +output "hosts" { + description = "Hostname, public IPv4, VPC IPv4 and (vpn hosts) WireGuard public key per host." + value = merge( + { for k, v in binarylane_server.edge : k => { + hostname = v.name + public_ipv4 = v.public_ipv4_addresses[0] + vpc_ipv4 = local.edges[k].vpc_ip + } }, + { for k, v in binarylane_server.vpn : k => { + hostname = v.name + public_ipv4 = v.public_ipv4_addresses[0] + vpc_ipv4 = local.vpns[k].vpc_ip + wg_address = local.vpns[k].wg_address + wg_pubkey = wireguard_asymmetric_key.vpn[k].public_key + } }, + ) } -output "wireguard" { - description = "Tunnel address and WireGuard public key per gateway, plus the prodnxsr0020 end." - value = { - hub = { address = local.hub_address, public_key = wireguard_asymmetric_key.hub.public_key, listen_port = local.hub_port } - gateways = { for k, v in local.gateways : k => { - address = v.wg_address - public_key = wireguard_asymmetric_key.gateway[k].public_key - } } - } +output "hub" { + description = "prodnxsr0020 end of the tunnels." + value = { address = local.hub_address, public_key = wireguard_asymmetric_key.hub.public_key, listen_port = local.hub_port } } From cc72e6c8d2064c920e810ffeb3747bc83c35a423 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 22:13:01 +1100 Subject: [PATCH 5/9] Route 198.18.0.0/16 and the management host through the vpn hosts --- modules/binarylane/main.tf | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 850fa8c..36ab260 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -8,17 +8,8 @@ locals { # lab destinations reached through each region's vpn host; SMTP stays on the public internet lab_routes = [ - "10.10.12.200/32", # management ssh host - "198.18.199.0/32", # git.unkin.net (rspamd rules) - "198.18.200.2/32", # puppetca.k8s.syd1.au.unkin.net - "198.18.200.3/32", # puppet.k8s.syd1.au.unkin.net - "198.18.200.4/32", # artifactapi.k8s.syd1.au.unkin.net - "198.18.200.7/32", # bind-internal resolvers - "198.18.25.78/32", # vault.service.consul - "198.18.26.197/32", # vault.service.consul - "198.18.27.116/32", # vault.service.consul - "198.18.28.248/32", # vault.service.consul - "198.18.29.92/32", # vault.service.consul + "198.18.0.0/16", + "10.10.12.200/32", # management ssh host ] vpcs = { From 29f3a531438dd1f1bbdc20c440a6c3b37c960e58 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 22:32:03 +1100 Subject: [PATCH 6/9] Give each vpn host its own p2p wg link to prodnxsr0020 --- README.md | 12 ++++--- modules/binarylane/cloud-init.yaml.tftpl | 6 ++-- modules/binarylane/main.tf | 46 ++++++++++++------------ modules/binarylane/outputs.tf | 15 ++++++-- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index f77953c..f84709c 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,10 @@ role in terraform-vault. ## Mail edge Each DMZ VPC (`syd`, `mel`) holds a postfix edge host and a vpn host. The vpn -host runs wg0 to prodnxsr0020 and forwards the lab routes for its VPC; the edge -host reaches the lab through it and has no WireGuard. wg0 is bootstrapped from -a key in `user_data` (readable via the BinaryLane API); Puppet then manages it -from Vault, so rotate by replacing `wireguard_asymmetric_key.vpn` here and -letting Puppet roll it out. +host runs wg0 as a /31 point-to-point link to its own prodnxsr0020 interface +(`wg-syd` on 51821, `wg-mel` on 51822; `router_links` output) and forwards the +lab routes for its VPC; the edge host reaches the lab through it and has no +WireGuard. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-`. +wg0 is bootstrapped from a key in `user_data` (readable via the BinaryLane +API); Puppet then manages it from Vault, so rotate by replacing +`wireguard_asymmetric_key.vpn` here and letting Puppet roll it out. diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl index 15cac91..e86da2b 100644 --- a/modules/binarylane/cloud-init.yaml.tftpl +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -9,13 +9,13 @@ write_files: permissions: "0600" content: | [Interface] - Address = ${vpn.address}/24 + Address = ${vpn.address} ListenPort = ${vpn.port} PrivateKey = ${vpn.private_key} [Peer] - PublicKey = ${vpn.hub_key} - Endpoint = ${vpn.hub} + PublicKey = ${vpn.router_key} + Endpoint = ${vpn.router} AllowedIPs = ${vpn.allowed_ips} PersistentKeepalive = 25 - path: /etc/sysctl.d/90-ip-forward.conf diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 36ab260..0f9ef76 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -1,10 +1,8 @@ locals { - # home gateway (prodnxsr0020) end of the mail DMZ tunnel - hub_endpoint = "103.216.191.185" - hub_port = 51821 - hub_address = "10.10.13.1" - wg_port = 51820 - domain = "main.unkin.net" + # home WAN address of prodnxsr0020, which terminates one wg link per vpn host + router_endpoint = "103.216.191.185" + wg_port = 51820 + domain = "main.unkin.net" # lab destinations reached through each region's vpn host; SMTP stays on the public internet lab_routes = [ @@ -18,8 +16,8 @@ locals { } vpns = { - ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", wg_address = "10.10.13.11" } - aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", wg_address = "10.10.13.12" } + ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", router_port = 51821, router_address = "10.10.13.0", wg_address = "10.10.13.1" } + aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", router_port = 51822, router_address = "10.10.13.2", wg_address = "10.10.13.3" } } edges = { @@ -46,20 +44,24 @@ resource "binarylane_vpc_route_entries" "dmz" { }] } -resource "wireguard_asymmetric_key" "hub" {} +resource "wireguard_asymmetric_key" "router" { + for_each = local.vpns +} resource "wireguard_asymmetric_key" "vpn" { for_each = local.vpns } -resource "vault_kv_secret_v2" "hub" { +resource "vault_kv_secret_v2" "router" { + for_each = local.vpns + mount = "kv" - name = "service/wireguard/binarylane/prodnxsr0020" + name = "service/wireguard/binarylane/prodnxsr0020-${each.value.region}" data_json = jsonencode({ - private_key = wireguard_asymmetric_key.hub.private_key - public_key = wireguard_asymmetric_key.hub.public_key - address = "${local.hub_address}/24" - listen_port = local.hub_port + private_key = wireguard_asymmetric_key.router[each.key].private_key + public_key = wireguard_asymmetric_key.router[each.key].public_key + address = "${each.value.router_address}/31" + listen_port = each.value.router_port }) } @@ -71,7 +73,7 @@ resource "vault_kv_secret_v2" "vpn" { data_json = jsonencode({ private_key = wireguard_asymmetric_key.vpn[each.key].private_key public_key = wireguard_asymmetric_key.vpn[each.key].public_key - address = "${each.value.wg_address}/24" + address = "${each.value.wg_address}/31" listen_port = local.wg_port }) } @@ -94,12 +96,12 @@ resource "binarylane_server" "vpn" { hostname = each.key fqdn = "${each.key}.${local.domain}" vpn = { - address = each.value.wg_address + address = "${each.value.wg_address}/31" port = local.wg_port private_key = wireguard_asymmetric_key.vpn[each.key].private_key - hub_key = wireguard_asymmetric_key.hub.public_key - hub = "${local.hub_endpoint}:${local.hub_port}" - allowed_ips = join(", ", concat(["${local.hub_address}/32"], local.lab_routes)) + router_key = wireguard_asymmetric_key.router[each.key].public_key + router = "${local.router_endpoint}:${each.value.router_port}" + allowed_ips = join(", ", concat(local.lab_routes, ["${each.value.router_address}/32"])) } routes = null }) @@ -197,9 +199,9 @@ resource "binarylane_server_firewall_rules" "vpn" { server_id = binarylane_server.vpn[each.key].id firewall_rules = [ { - description = "wireguard from home gateway" + description = "wireguard from prodnxsr0020" protocol = "udp" - source_addresses = ["${local.hub_endpoint}/32"] + source_addresses = ["${local.router_endpoint}/32"] destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip] destination_ports = [tostring(local.wg_port)] action = "accept" diff --git a/modules/binarylane/outputs.tf b/modules/binarylane/outputs.tf index a45f9ea..e04854e 100644 --- a/modules/binarylane/outputs.tf +++ b/modules/binarylane/outputs.tf @@ -16,7 +16,16 @@ output "hosts" { ) } -output "hub" { - description = "prodnxsr0020 end of the tunnels." - value = { address = local.hub_address, public_key = wireguard_asymmetric_key.hub.public_key, listen_port = local.hub_port } +output "router_links" { + description = "Per-region p2p wg link as prodnxsr0020 sees it; consumed by its Puppet config." + value = { for k, v in local.vpns : v.region => { + interface = "wg-${v.region}" + listen_port = v.router_port + router_address = "${v.router_address}/31" + peer_address = v.wg_address + peer_public_key = wireguard_asymmetric_key.vpn[k].public_key + peer_endpoint = "${binarylane_server.vpn[k].public_ipv4_addresses[0]}:${local.wg_port}" + vpc_cidr = local.vpcs[v.region] + private_key_vault_path = "kv/${vault_kv_secret_v2.router[k].name}" + } } } From 04f6fd69d342bab6bfd8afb2a0b6d15c7ddf7cd8 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 22:39:17 +1100 Subject: [PATCH 7/9] Address vpn wg links and loopbacks from 198.18.2.160/27 for ospf --- README.md | 4 +++- modules/binarylane/cloud-init.yaml.tftpl | 21 +++++++++++++++++++++ modules/binarylane/main.tf | 12 +++++++++--- modules/binarylane/outputs.tf | 10 +++++++--- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f84709c..ca65c74 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,9 @@ Each DMZ VPC (`syd`, `mel`) holds a postfix edge host and a vpn host. The vpn host runs wg0 as a /31 point-to-point link to its own prodnxsr0020 interface (`wg-syd` on 51821, `wg-mel` on 51822; `router_links` output) and forwards the lab routes for its VPC; the edge host reaches the lab through it and has no -WireGuard. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-`. +WireGuard. Link /31s and vpn loopbacks (dum0) come from 198.18.2.160/27. Lab +traffic is sourced from the /31 until Puppet brings up FRR, which peers OSPF +over wg0 and advertises the loopback and VPC to the router. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-`. wg0 is bootstrapped from a key in `user_data` (readable via the BinaryLane API); Puppet then manages it from Vault, so rotate by replacing `wireguard_asymmetric_key.vpn` here and letting Puppet roll it out. diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl index e86da2b..3dbab72 100644 --- a/modules/binarylane/cloud-init.yaml.tftpl +++ b/modules/binarylane/cloud-init.yaml.tftpl @@ -12,12 +12,31 @@ write_files: Address = ${vpn.address} ListenPort = ${vpn.port} PrivateKey = ${vpn.private_key} + # source lab traffic from the /31: the router only learns the loopback and vpc once ospf is up + Table = off +%{ for r in vpn.routes ~} + PostUp = ip route replace ${r} dev wg0 src ${vpn.src} +%{ endfor ~} [Peer] PublicKey = ${vpn.router_key} Endpoint = ${vpn.router} AllowedIPs = ${vpn.allowed_ips} PersistentKeepalive = 25 + - path: /etc/NetworkManager/system-connections/dum0.nmconnection + permissions: "0600" + content: | + [connection] + id=dum0 + type=dummy + interface-name=dum0 + + [ipv4] + method=manual + address1=${vpn.loopback} + + [ipv6] + method=disabled - path: /etc/sysctl.d/90-ip-forward.conf content: | net.ipv4.ip_forward = 1 @@ -39,6 +58,8 @@ write_files: %{ if vpn != null ~} rpm -q wireguard-tools >/dev/null || dnf install -y wireguard-tools sysctl -p /etc/sysctl.d/90-ip-forward.conf + nmcli connection reload + nmcli connection up dum0 systemctl enable --now wg-quick@wg0 wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }' %{ endif ~} diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 0f9ef76..86b598b 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -10,14 +10,17 @@ locals { "10.10.12.200/32", # management ssh host ] + # ospf hellos over the p2p links; wg has one peer per interface so this routes to it + ospf_allspf = "224.0.0.5/32" + vpcs = { syd = "10.10.14.0/24" mel = "10.10.15.0/24" } vpns = { - ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", router_port = 51821, router_address = "10.10.13.0", wg_address = "10.10.13.1" } - aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", router_port = 51822, router_address = "10.10.13.2", wg_address = "10.10.13.3" } + ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", router_port = 51821, router_address = "198.18.2.176", wg_address = "198.18.2.177", loopback = "198.18.2.161" } + aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", router_port = 51822, router_address = "198.18.2.178", wg_address = "198.18.2.179", loopback = "198.18.2.162" } } edges = { @@ -97,11 +100,14 @@ resource "binarylane_server" "vpn" { fqdn = "${each.key}.${local.domain}" vpn = { address = "${each.value.wg_address}/31" + loopback = "${each.value.loopback}/32" + routes = local.lab_routes + src = each.value.wg_address port = local.wg_port private_key = wireguard_asymmetric_key.vpn[each.key].private_key router_key = wireguard_asymmetric_key.router[each.key].public_key router = "${local.router_endpoint}:${each.value.router_port}" - allowed_ips = join(", ", concat(local.lab_routes, ["${each.value.router_address}/32"])) + allowed_ips = join(", ", concat(local.lab_routes, [local.ospf_allspf])) } routes = null }) diff --git a/modules/binarylane/outputs.tf b/modules/binarylane/outputs.tf index e04854e..0bf347d 100644 --- a/modules/binarylane/outputs.tf +++ b/modules/binarylane/outputs.tf @@ -1,5 +1,5 @@ output "hosts" { - description = "Hostname, public IPv4, VPC IPv4 and (vpn hosts) WireGuard public key per host." + description = "Hostname, public IPv4, VPC IPv4 and (vpn hosts) wg address, loopback and public key per host." value = merge( { for k, v in binarylane_server.edge : k => { hostname = v.name @@ -11,21 +11,25 @@ output "hosts" { public_ipv4 = v.public_ipv4_addresses[0] vpc_ipv4 = local.vpns[k].vpc_ip wg_address = local.vpns[k].wg_address + loopback = local.vpns[k].loopback wg_pubkey = wireguard_asymmetric_key.vpn[k].public_key } }, ) } output "router_links" { - description = "Per-region p2p wg link as prodnxsr0020 sees it; consumed by its Puppet config." + description = "Per-region p2p wg/ospf link as prodnxsr0020 sees it; vpc_cidr arrives via ospf. Consumed by its Puppet config." value = { for k, v in local.vpns : v.region => { interface = "wg-${v.region}" listen_port = v.router_port router_address = "${v.router_address}/31" - peer_address = v.wg_address + peer_address = "${v.wg_address}/31" + peer_loopback = "${v.loopback}/32" + allowed_ips = ["${v.wg_address}/32", "${v.loopback}/32", local.vpcs[v.region], local.ospf_allspf] peer_public_key = wireguard_asymmetric_key.vpn[k].public_key peer_endpoint = "${binarylane_server.vpn[k].public_ipv4_addresses[0]}:${local.wg_port}" vpc_cidr = local.vpcs[v.region] private_key_vault_path = "kv/${vault_kv_secret_v2.router[k].name}" + peer_key_vault_path = "kv/${vault_kv_secret_v2.vpn[k].name}" } } } From 6a0b6989aec1af79c120f246e050b329cbc56861 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Tue, 13 Oct 2026 21:29:07 +1100 Subject: [PATCH 8/9] Number the DMZ VPCs 198.18.30.0/24 (syd) and 198.18.31.0/24 (mel) --- README.md | 5 +++-- modules/binarylane/main.tf | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ca65c74..cc64429 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,12 @@ role in terraform-vault. ## Mail edge -Each DMZ VPC (`syd`, `mel`) holds a postfix edge host and a vpn host. The vpn +Each DMZ VPC (`syd` 198.18.30.0/24, `mel` 198.18.31.0/24) holds a postfix edge host and a vpn host. The vpn host runs wg0 as a /31 point-to-point link to its own prodnxsr0020 interface (`wg-syd` on 51821, `wg-mel` on 51822; `router_links` output) and forwards the lab routes for its VPC; the edge host reaches the lab through it and has no -WireGuard. Link /31s and vpn loopbacks (dum0) come from 198.18.2.160/27. Lab +WireGuard. The VPC /24 is connected on both hosts, so it wins over the lab +198.18.0.0/16 route via the vpn host. Link /31s and vpn loopbacks (dum0) come from 198.18.2.160/27. Lab traffic is sourced from the /31 until Puppet brings up FRR, which peers OSPF over wg0 and advertises the loopback and VPC to the router. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-`. wg0 is bootstrapped from a key in `user_data` (readable via the BinaryLane diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 86b598b..6d665f4 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -14,18 +14,18 @@ locals { ospf_allspf = "224.0.0.5/32" vpcs = { - syd = "10.10.14.0/24" - mel = "10.10.15.0/24" + syd = "198.18.30.0/24" + mel = "198.18.31.0/24" } vpns = { - ausyd2nxvm2131 = { region = "syd", vpc_ip = "10.10.14.11", router_port = 51821, router_address = "198.18.2.176", wg_address = "198.18.2.177", loopback = "198.18.2.161" } - aumel1nxvm2133 = { region = "mel", vpc_ip = "10.10.15.11", router_port = 51822, router_address = "198.18.2.178", wg_address = "198.18.2.179", loopback = "198.18.2.162" } + ausyd2nxvm2131 = { region = "syd", vpc_ip = "198.18.30.11", router_port = 51821, router_address = "198.18.2.176", wg_address = "198.18.2.177", loopback = "198.18.2.161" } + aumel1nxvm2133 = { region = "mel", vpc_ip = "198.18.31.11", router_port = 51822, router_address = "198.18.2.178", wg_address = "198.18.2.179", loopback = "198.18.2.162" } } edges = { - ausyd2nxvm2130 = { region = "syd", vpc_ip = "10.10.14.10", vpn = "ausyd2nxvm2131", ptr = "mta1.edge.unkin.net" } - aumel1nxvm2132 = { region = "mel", vpc_ip = "10.10.15.10", vpn = "aumel1nxvm2133", ptr = "mta2.edge.unkin.net" } + ausyd2nxvm2130 = { region = "syd", vpc_ip = "198.18.30.10", vpn = "ausyd2nxvm2131", ptr = "mta1.edge.unkin.net" } + aumel1nxvm2132 = { region = "mel", vpc_ip = "198.18.31.10", vpn = "aumel1nxvm2133", ptr = "mta2.edge.unkin.net" } } } From 88f50fc46a2358a4942d1f9da1d1dcc87b4e0405 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Tue, 13 Oct 2026 21:34:44 +1100 Subject: [PATCH 9/9] Keep VPC route entries clear of the VPC ranges --- modules/binarylane/main.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf index 6d665f4..ce1d9e4 100644 --- a/modules/binarylane/main.tf +++ b/modules/binarylane/main.tf @@ -10,6 +10,17 @@ locals { "10.10.12.200/32", # management ssh host ] + # vpc route entries must not overlap the vpcs themselves: the /16 minus 198.18.30.0/23 + vpc_lab_routes = concat([ + "198.18.0.0/20", + "198.18.16.0/21", + "198.18.24.0/22", + "198.18.28.0/23", + "198.18.32.0/19", + "198.18.64.0/18", + "198.18.128.0/17", + ], [for dst in local.lab_routes : dst if dst != "198.18.0.0/16"]) + # ospf hellos over the p2p links; wg has one peer per interface so this routes to it ospf_allspf = "224.0.0.5/32" @@ -40,7 +51,7 @@ resource "binarylane_vpc_route_entries" "dmz" { for_each = local.vpns vpc_id = binarylane_vpc.dmz[each.value.region].id - route_entries = [for dst in local.lab_routes : { + route_entries = [for dst in local.vpc_lab_routes : { description = "lab via ${each.key}" destination = dst router = binarylane_server.vpn[each.key].private_ipv4_addresses[0]