From 975ff15b6bb498f57eaa4a9f6b4a56915d8e0594 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Mon, 12 Oct 2026 21:57:40 +1100
Subject: [PATCH] Bootstrap gateways via a retrying wg handshake unit and guard
 servers against rebuilds

---
 README.md                                |  6 +++
 environments/global/.terraform.lock.hcl  | 50 ------------------------
 modules/binarylane/cloud-init.yaml.tftpl | 37 ++++++++++++++----
 modules/binarylane/main.tf               |  6 +++
 4 files changed, 41 insertions(+), 58 deletions(-)
 delete mode 100644 environments/global/.terraform.lock.hcl

diff --git a/README.md b/README.md
index b2069fb..63966c6 100644
--- a/README.md
+++ b/README.md
@@ -20,3 +20,9 @@ The BinaryLane API token is read from `kv/service/terraform/binarylane` (field
 from `consul_root/au/syd1/creds/terraform-binarylane`. Both are granted to the
 `terraform_binarylane` AppRole and the `woodpecker_terraform_binarylane` k8s
 role in terraform-vault.
+
+## Gateways
+
+`mta1`/`mta2` bootstrap wg0 from a key in `user_data` (readable via the
+BinaryLane API); Puppet then manages wg0 from Vault, so rotate by replacing
+`wireguard_asymmetric_key.gateway` here and letting Puppet roll it out.
diff --git a/environments/global/.terraform.lock.hcl b/environments/global/.terraform.lock.hcl
deleted file mode 100644
index 39ba811..0000000
--- a/environments/global/.terraform.lock.hcl
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file is maintained automatically by "tofu init".
-# Manual edits may be lost in future updates.
-
-provider "artifactapi.k8s.syd1.au.unkin.net/terraform-unkin/binarylane" {
-  version     = "0.14.0"
-  constraints = "0.14.0"
-  hashes = [
-    "h1:k72XOG+O2dyqOXoFYzdaOYzIBELekwNF1XRlCjZaSa0=",
-    "zh:d84c05d191a0c53c96b8fbac62f8ccd2d41678f008406dd656d29d2c10766b45",
-  ]
-}
-
-provider "registry.opentofu.org/hashicorp/vault" {
-  version     = "5.6.0"
-  constraints = "5.6.0"
-  hashes = [
-    "h1:afl/41BzHl93OACbk2EnGHF/xVgHtzmEREcdejASJcM=",
-    "zh:437c9f3920cc29d7a453aa5342270d55c618ca4979718c34641a49af1437d5a8",
-    "zh:5aeec1c3ed1710d5594277250ae4202c97bf9d8462a3c672c689e96bec3d7c8a",
-    "zh:71a61b7667e1016b5ed524b5842c65be1ab1661f258faf7a7cec9db4ea44abd8",
-    "zh:769a946cc4d99752dce8db0c4e509a201c0063f266dd387164b239854392d1b7",
-    "zh:82b0173dc9f65035b4ae6faaea497239455e5562d5d8c24a1f882e4451234dcf",
-    "zh:8f3373ef3dd1e026424c194d46842c0451ee79aacc67d1a1087d8d1ef8e55e95",
-    "zh:984d15ebd2fdaa55608efff7d98d59e414a1cb8c7b899b883fbbfbeb5849b70d",
-    "zh:9b4194ab7cf28d22098c51436dbfea878e413c418d6dbcfd3689aeb580993e21",
-    "zh:ef30a4958328b9fa3d58845ddebe8bd574a533170a28dc7b73121d52f565563b",
-  ]
-}
-
-provider "registry.opentofu.org/ojford/wireguard" {
-  version     = "0.4.1"
-  constraints = "0.4.1"
-  hashes = [
-    "h1:c09xyC2LPfUHgVG/nis4l1B51rBZOCJwfQIcn5RJ3/k=",
-    "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7",
-    "zh:1cdcd10ba61520d7e21590ea4d96d188cefbce3dcdbf89b277696d6023ec7620",
-    "zh:26bde3cf4850ba8605c028c84bed6075d2175853a0744303f6051cc3b5c51dd1",
-    "zh:28c351481689762dcd653229aa5f847473f71568a8222fafbddda0c600377a46",
-    "zh:3bd7ab222f13286449342d35d6dd3dbe343f575e47d4c2e6f8f64149386070c7",
-    "zh:3d25381c96df52253e7767aae49f0bead427c79f34ecdfb38026335e03c12de2",
-    "zh:4017a40b50886d3f1f0e886f8eacd1924c5f03888c90862c55ba98bb376198b2",
-    "zh:561978966998a3703053ac990796d5a88c7c14afd77e623846941a66b6fc9c92",
-    "zh:5f8460880a5f9cc9f4cc7c7c3c41156b0dae3c2b56f29cc259b3dc75ad650fe1",
-    "zh:724592993a0cccd45e0290aba2a6b72f7bb668a4ae3b8d778bb5d379d4105863",
-    "zh:b4de1157176eaf1fcc624d812ae6134893d6c73f3d1d2325c94cd9e707678d7b",
-    "zh:c5530bc71beedba579be57ea580f4151ec87223f198147771a924debff2e255e",
-    "zh:d3c7863e5baaf05e76fa5db3d3f58ff94440408873e370770b6a0080981cec37",
-    "zh:ec0a7d3a540cfb92e15b9a7acc43446059a9254b5904b48c940564c70e2b8aa1",
-  ]
-}
diff --git a/modules/binarylane/cloud-init.yaml.tftpl b/modules/binarylane/cloud-init.yaml.tftpl
index 78cbd88..c362ed2 100644
--- a/modules/binarylane/cloud-init.yaml.tftpl
+++ b/modules/binarylane/cloud-init.yaml.tftpl
@@ -17,21 +17,42 @@ write_files:
       Endpoint = ${hub_endpoint}
       AllowedIPs = ${allowed_ips}
       PersistentKeepalive = 25
-  - path: /etc/NetworkManager/conf.d/90-dns-none.conf
+  - path: /usr/local/sbin/wg-bootstrap
+    permissions: "0755"
     content: |
-      [main]
-      dns=none
+      #!/bin/sh
+      set -e
+      wg show wg0 latest-handshakes | awk '$2 > 0 { ok = 1 } END { exit !ok }'
+      printf '[main]\ndns=none\n' > /etc/NetworkManager/conf.d/90-dns-none.conf
+      systemctl reload NetworkManager
+      printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf
+      systemctl enable --now puppet
+      systemctl disable wg-bootstrap
+  - path: /etc/systemd/system/wg-bootstrap.service
+    content: |
+      [Unit]
+      Description=Switch to internal DNS and start puppet once wg0 has a handshake
+      After=wg-quick@wg0.service
+      StartLimitIntervalSec=0
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/local/sbin/wg-bootstrap
+      Restart=on-failure
+      RestartSec=30
+
+      [Install]
+      WantedBy=multi-user.target
 
 runcmd:
   - dnf install -y wireguard-tools
   - systemctl enable --now wg-quick@wg0
-  - systemctl reload NetworkManager
-  - printf 'search main.unkin.net\nnameserver 198.18.200.7\n' > /etc/resolv.conf
-  - rpm -q puppet-agent || dnf install -y https://yum.puppet.com/puppet8-release-el-9.noarch.rpm
-  - dnf install -y puppet-agent
+  - rpm -q openvox-agent || dnf install -y https://yum.voxpupuli.org/openvox7-release-el-9.noarch.rpm
+  - dnf install -y openvox-agent
   - /opt/puppetlabs/bin/puppet config set --section main certname ${fqdn}
   - /opt/puppetlabs/bin/puppet config set --section main server puppet.k8s.syd1.au.unkin.net
   - /opt/puppetlabs/bin/puppet config set --section main ca_server puppetca.k8s.syd1.au.unkin.net
   - /opt/puppetlabs/bin/puppet config set --section main report_server puppet.k8s.syd1.au.unkin.net
   - /opt/puppetlabs/bin/puppet config set --section main environment production
-  - systemctl enable --now puppet
+  - systemctl daemon-reload
+  - systemctl enable --now --no-block wg-bootstrap
diff --git a/modules/binarylane/main.tf b/modules/binarylane/main.tf
index 02162c7..1ac1bce 100644
--- a/modules/binarylane/main.tf
+++ b/modules/binarylane/main.tf
@@ -92,6 +92,12 @@ resource "binarylane_server" "gateway" {
     hub_endpoint   = "${local.hub_endpoint}:${local.hub_port}"
     allowed_ips    = join(", ", local.wg_allowed_ips)
   })
+
+  # user_data/image changes reinstall the server in place, wiping its disk
+  lifecycle {
+    ignore_changes  = [user_data, image]
+    prevent_destroy = true
+  }
 }
 
 resource "binarylane_server_reverse_name" "gateway" {