unkin/terraform-binarylane
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'Add mail edge and vpn hosts to the DMZ VPCs' (#2) from benvin/mail-gateways into main
ci/woodpecker/push/apply Pipeline failed
Details

Browse Source 
Reviewed-on: #2
This commit was merged in pull request #2.
This commit is contained in:
BenVincent
2026-10-14 22:17:03 +11:00
parent f5921bf4c8 88f50fc46a
commit a8a2f50491
8 changed files with 470 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+14
View File
@@ -20,3 +20,17 @@ The BinaryLane API token is read from `kv/service/terraform/binarylane` (field
from `consul_root/au/syd1/creds/terraform-binarylane`. Both are granted to the
`terraform_binarylane` AppRole and the `woodpecker_terraform_binarylane` k8s
role in terraform-vault.
## Mail edge
Each DMZ VPC (`syd` 198.18.30.0/24, `mel` 198.18.31.0/24) holds a postfix edge host and a vpn host. The vpn
host runs wg0 as a /31 point-to-point link to its own prodnxsr0020 interface
(`wg-syd` on 51821, `wg-mel` on 51822; `router_links` output) and forwards the
lab routes for its VPC; the edge host reaches the lab through it and has no
WireGuard. The VPC /24 is connected on both hosts, so it wins over the lab
198.18.0.0/16 route via the vpn host. Link /31s and vpn loopbacks (dum0) come from 198.18.2.160/27. Lab
traffic is sourced from the /31 until Puppet brings up FRR, which peers OSPF
over wg0 and advertises the loopback and VPC to the router. Router keys live at `kv/service/wireguard/binarylane/prodnxsr0020-<region>`.
wg0 is bootstrapped from a key in `user_data` (readable via the BinaryLane
API); Puppet then manages it from Vault, so rotate by replacing
`wireguard_asymmetric_key.vpn` here and letting Puppet roll it out.
modules/binarylane/.terraform.lock.hcl
Generated
+50
View File
@@ -0,0 +1,50 @@
# This file is maintained automatically by "tofu init".
# Manual edits may be lost in future updates.
provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" {
version = "0.14.0"
constraints = "0.14.0"
hashes = [
"h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=",
"zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45",
]
}
provider "registry.opentofu.org/hashicorp/vault" {
version = "5.6.0"
constraints = "5.6.0"
hashes = [
"h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=",
"zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8",
"zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a",
"zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8",
"zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7",
"zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf",
"zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95",
"zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d",
"zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21",
"zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b",
]
}
provider "registry.opentofu.org/ojford/wireguard" {
version = "0.4.1"
constraints = "0.4.1"
hashes = [
"h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=",
"zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7",
"zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620",
"zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1",
"zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46",
"zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7",
"zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2",
"zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2",
"zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92",
"zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1",
"zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863",
"zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b",
"zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e",
"zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37",
"zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1",
]
}
modules/binarylane/.tflint.hcl
-3
View File
@@ -1,3 +0,0 @@
rule "terraform_unused_declarations" {
enabled = false
}
modules/binarylane/cloud-init.yaml.tftpl
+100
View File
@@ -0,0 +1,100 @@
#cloud-config
hostname: ${hostname}
fqdn: ${fqdn}
prefer_fqdn_over_hostname: true
write_files:
%{ if vpn != null ~}
- path: /etc/wireguard/wg0.conf
permissions: "0600"
content: |
[Interface]
Address = ${vpn.address}
ListenPort = ${vpn.port}
PrivateKey = ${vpn.private_key}
# source lab traffic from the /31: the router only learns the loopback and vpc once ospf is up
Table = off
%{ for r in vpn.routes ~}
PostUp = ip route replace ${r} dev wg0 src ${vpn.src}
%{ endfor ~}
[Peer]
PublicKey = ${vpn.router_key}
Endpoint = ${vpn.router}
AllowedIPs = ${vpn.allowed_ips}
PersistentKeepalive = 25
- path: /etc/NetworkManager/system-connections/dum0.nmconnection
permissions: "0600"
content: |
[connection]
id=dum0
type=dummy
interface-name=dum0
[ipv4]
method=manual
address1=${vpn.loopback}
[ipv6]
method=disabled
- path: /etc/sysctl.d/90-ip-forward.conf
content: |
net.ipv4.ip_forward = 1
%{ endif ~}
- path: /usr/local/sbin/lab-bootstrap
permissions: "0755"
content: |
#!/bin/bash
set -euo pipefail
if ! rpm -q openvox-agent >/dev/null; then
rpm -q openvox7-release >/dev/null || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm
dnf install -y openvox-agent
fi
/opt/puppetlabs/bin/puppet config set --section main certname ${fqdn}
/opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net
/opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net
/opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net
/opt/puppetlabs/bin/puppet config set --section main environment production
%{ if vpn != null ~}
rpm -q wireguard-tools >/dev/null || dnf install -y wireguard-tools
sysctl -p /etc/sysctl.d/90-ip-forward.conf
nmcli connection reload
nmcli connection up dum0
systemctl enable --now wg-quick@wg0
wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }'
%{ endif ~}
%{ if routes != null ~}
if ! ip route show 198.18.200.3 | grep -q .; then
dev=$(ip -o -4 addr show to ${routes.vpc} | awk '{ print $2; exit }')
con=$(nmcli -g GENERAL.CONNECTION device show "$dev")
nmcli connection modify "$con" ipv4.routes "${routes.list}"
nmcli device reapply "$dev"
fi
%{ endif ~}
timeout 5 bash -c '</dev/tcp/198.18.200.3/8140'
printf '[main]\ndns=none\n' > /etc/NetworkManager/conf.d/90-dns-none.conf
nmcli general reload
rm -f /etc/resolv.conf
printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf
systemctl enable --now puppet
systemctl disable lab-bootstrap
- path: /etc/systemd/system/lab-bootstrap.service
content: |
[Unit]
Description=Install openvox, wait for the lab via the vpn host, then switch to internal DNS and start puppet
Wants=network-online.target
After=network-online.target
StartLimitIntervalSec=0
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/lab-bootstrap
Restart=on-failure
RestartSec=30
[Install]
WantedBy=multi-user.target
runcmd:
- systemctl daemon-reload
- systemctl enable --now --no-block lab-bootstrap
modules/binarylane/main.tf
+248
View File
@@ -0,0 +1,248 @@
locals {
# home WAN address of prodnxsr0020, which terminates one wg link per vpn host
router_endpoint = "103.216.191.185"
wg_port = 51820
domain = "main.unkin.net"
# lab destinations reached through each region's vpn host; SMTP stays on the public internet
lab_routes = [
"198.18.0.0/16",
"10.10.12.200/32", # management ssh host
]
# vpc route entries must not overlap the vpcs themselves: the /16 minus 198.18.30.0/23
vpc_lab_routes = concat([
"198.18.0.0/20",
"198.18.16.0/21",
"198.18.24.0/22",
"198.18.28.0/23",
"198.18.32.0/19",
"198.18.64.0/18",
"198.18.128.0/17",
], [for dst in local.lab_routes : dst if dst != "198.18.0.0/16"])
# ospf hellos over the p2p links; wg has one peer per interface so this routes to it
ospf_allspf = "224.0.0.5/32"
vpcs = {
syd = "198.18.30.0/24"
mel = "198.18.31.0/24"
}
vpns = {
ausyd2nxvm2131 = { region = "syd", vpc_ip = "198.18.30.11", router_port = 51821, router_address = "198.18.2.176", wg_address = "198.18.2.177", loopback = "198.18.2.161" }
aumel1nxvm2133 = { region = "mel", vpc_ip = "198.18.31.11", router_port = 51822, router_address = "198.18.2.178", wg_address = "198.18.2.179", loopback = "198.18.2.162" }
}
edges = {
ausyd2nxvm2130 = { region = "syd", vpc_ip = "198.18.30.10", vpn = "ausyd2nxvm2131", ptr = "mta1.edge.unkin.net" }
aumel1nxvm2132 = { region = "mel", vpc_ip = "198.18.31.10", vpn = "aumel1nxvm2133", ptr = "mta2.edge.unkin.net" }
}
}
resource "binarylane_vpc" "dmz" {
for_each = local.vpcs
name = "dmz-${each.key}"
ip_range = each.value
}
resource "binarylane_vpc_route_entries" "dmz" {
for_each = local.vpns
vpc_id = binarylane_vpc.dmz[each.value.region].id
route_entries = [for dst in local.vpc_lab_routes : {
description = "lab via ${each.key}"
destination = dst
router = binarylane_server.vpn[each.key].private_ipv4_addresses[0]
}]
}
resource "wireguard_asymmetric_key" "router" {
for_each = local.vpns
}
resource "wireguard_asymmetric_key" "vpn" {
for_each = local.vpns
}
resource "vault_kv_secret_v2" "router" {
for_each = local.vpns
mount = "kv"
name = "service/wireguard/binarylane/prodnxsr0020-${each.value.region}"
data_json = jsonencode({
private_key = wireguard_asymmetric_key.router[each.key].private_key
public_key = wireguard_asymmetric_key.router[each.key].public_key
address = "${each.value.router_address}/31"
listen_port = each.value.router_port
})
}
resource "vault_kv_secret_v2" "vpn" {
for_each = local.vpns
mount = "kv"
name = "service/wireguard/binarylane/${each.key}"
data_json = jsonencode({
private_key = wireguard_asymmetric_key.vpn[each.key].private_key
public_key = wireguard_asymmetric_key.vpn[each.key].public_key
address = "${each.value.wg_address}/31"
listen_port = local.wg_port
})
}
resource "binarylane_server" "vpn" {
for_each = local.vpns
name = "${each.key}.${local.domain}"
region = each.value.region
image = "alma-9"
size = "std-min"
public_ipv4_count = 1
vpc_id = binarylane_vpc.dmz[each.value.region].id
vpc_ipv4_address = each.value.vpc_ip
source_and_destination_check = false
backups = false
ipv6 = false
user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", {
hostname = each.key
fqdn = "${each.key}.${local.domain}"
vpn = {
address = "${each.value.wg_address}/31"
loopback = "${each.value.loopback}/32"
routes = local.lab_routes
src = each.value.wg_address
port = local.wg_port
private_key = wireguard_asymmetric_key.vpn[each.key].private_key
router_key = wireguard_asymmetric_key.router[each.key].public_key
router = "${local.router_endpoint}:${each.value.router_port}"
allowed_ips = join(", ", concat(local.lab_routes, [local.ospf_allspf]))
}
routes = null
})
# user_data/image changes reinstall the server in place, wiping its disk
lifecycle {
ignore_changes = [user_data, image]
prevent_destroy = true
}
}
resource "binarylane_server" "edge" {
for_each = local.edges
name = "${each.key}.${local.domain}"
region = each.value.region
image = "alma-9"
size = "std-min"
public_ipv4_count = 1
vpc_id = binarylane_vpc.dmz[each.value.region].id
vpc_ipv4_address = each.value.vpc_ip
port_blocking = false
backups = false
ipv6 = false
user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", {
hostname = each.key
fqdn = "${each.key}.${local.domain}"
vpn = null
routes = {
vpc = local.vpcs[each.value.region]
list = join(", ", [
for dst in local.lab_routes : "${dst} ${local.vpns[each.value.vpn].vpc_ip} src=${each.value.vpc_ip}"
])
}
})
# user_data/image changes reinstall the server in place, wiping its disk
lifecycle {
ignore_changes = [user_data, image]
prevent_destroy = true
}
}
resource "binarylane_server_reverse_name" "edge" {
for_each = local.edges
server_id = binarylane_server.edge[each.key].id
ipv4_address = binarylane_server.edge[each.key].public_ipv4_addresses[0]
reverse_name = each.value.ptr
}
resource "binarylane_server_firewall_rules" "edge" {
for_each = local.edges
server_id = binarylane_server.edge[each.key].id
firewall_rules = [
{
description = "smtp"
protocol = "tcp"
source_addresses = ["0.0.0.0/0"]
destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
destination_ports = ["25"]
action = "accept"
},
{
description = "ssh from the lab via ${each.value.vpn}"
protocol = "tcp"
source_addresses = ["${local.vpns[each.value.vpn].vpc_ip}/32", "10.10.12.200/32"]
destination_addresses = [each.value.vpc_ip]
destination_ports = ["22"]
action = "accept"
},
{
description = "icmp"
protocol = "icmp"
source_addresses = ["0.0.0.0/0"]
destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
action = "accept"
},
{
description = "default deny"
protocol = "all"
source_addresses = ["0.0.0.0/0"]
destination_addresses = [binarylane_server.edge[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
action = "drop"
},
]
}
# SSH has no rule: management reaches sshd through wg0, which this firewall never sees.
resource "binarylane_server_firewall_rules" "vpn" {
for_each = local.vpns
server_id = binarylane_server.vpn[each.key].id
firewall_rules = [
{
description = "wireguard from prodnxsr0020"
protocol = "udp"
source_addresses = ["${local.router_endpoint}/32"]
destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
destination_ports = [tostring(local.wg_port)]
action = "accept"
},
{
description = "forward dmz to lab"
protocol = "all"
source_addresses = [local.vpcs[each.value.region]]
destination_addresses = local.lab_routes
action = "accept"
},
{
description = "icmp"
protocol = "icmp"
source_addresses = ["0.0.0.0/0"]
destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
action = "accept"
},
{
description = "default deny"
protocol = "all"
source_addresses = ["0.0.0.0/0"]
destination_addresses = [binarylane_server.vpn[each.key].public_ipv4_addresses[0], each.value.vpc_ip]
action = "drop"
},
]
}
modules/binarylane/outputs.tf
+35
View File
@@ -0,0 +1,35 @@
output "hosts" {
description = "Hostname, public IPv4, VPC IPv4 and (vpn hosts) wg address, loopback and public key per host."
value = merge(
{ for k, v in binarylane_server.edge : k => {
hostname = v.name
public_ipv4 = v.public_ipv4_addresses[0]
vpc_ipv4 = local.edges[k].vpc_ip
} },
{ for k, v in binarylane_server.vpn : k => {
hostname = v.name
public_ipv4 = v.public_ipv4_addresses[0]
vpc_ipv4 = local.vpns[k].vpc_ip
wg_address = local.vpns[k].wg_address
loopback = local.vpns[k].loopback
wg_pubkey = wireguard_asymmetric_key.vpn[k].public_key
} },
)
}
output "router_links" {
description = "Per-region p2p wg/ospf link as prodnxsr0020 sees it; vpc_cidr arrives via ospf. Consumed by its Puppet config."
value = { for k, v in local.vpns : v.region => {
interface = "wg-${v.region}"
listen_port = v.router_port
router_address = "${v.router_address}/31"
peer_address = "${v.wg_address}/31"
peer_loopback = "${v.loopback}/32"
allowed_ips = ["${v.wg_address}/32", "${v.loopback}/32", local.vpcs[v.region], local.ospf_allspf]
peer_public_key = wireguard_asymmetric_key.vpn[k].public_key
peer_endpoint = "${binarylane_server.vpn[k].public_ipv4_addresses[0]}:${local.wg_port}"
vpc_cidr = local.vpcs[v.region]
private_key_vault_path = "kv/${vault_kv_secret_v2.router[k].name}"
peer_key_vault_path = "kv/${vault_kv_secret_v2.vpn[k].name}"
} }
}
modules/binarylane/providers.tf
+8
View File
@@ -0,0 +1,8 @@
provider "binarylane" {
api_token = var.binarylane_token
}
# VAULT_ADDR / VAULT_TOKEN come from the Makefile; the k8s auth role token cannot mint child tokens.
provider "vault" {
skip_child_token = true
}
modules/binarylane/versions.tf
+15
View File
@@ -1,3 +1,18 @@
terraform {
required_version = ">= 1.10"
required_providers {
binarylane = {
source = "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane"
version = "0.14.0"
}
vault = {
source = "hashicorp/vault"
version = "5.6.0"
}
wireguard = {
source = "OJFord/wireguard"
version = "0.4.1"
}
}
}