dns: dual-write toggles + drift fact

Publish records both ways during the k8s cutover, and expose expected vs
deployed records for drift detection.

- profiles::dns::updater + ::record: manage_nsupdate and manage_export
  booleans (both default on); export keeps the legacy master flow, so
  disable it once k8s is authoritative
- dns_records fact: parses the expected records file and digs the
  authoritative server for each, reporting expected / in_sync / drift
  (plus dns_records_insync boolean); updater writes the server address
  to /var/lib/dns-updater/server for the fact
- hiera: manage_export/manage_nsupdate = true (cutover)

.woodpecker/bolt-validate.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/epp-validate.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/erb-validate.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/puppet-lint.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/puppet-validate.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 1
+            cpu: 2
           limits:
             memory: 2Gi
             cpu: 2

.woodpecker/ruby-check.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/ruby-validate.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

.woodpecker/yamllint.yaml

@@ -12,7 +12,7 @@ steps:
         resources:
           requests:
             memory: 512Mi
-            cpu: 250m
+            cpu: 1
           limits:
             memory: 2Gi
             cpu: 1

hieradata/common.eyaml

@@ -6,4 +6,3 @@ profiles::consul::token::node_editor::secret_id: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDo
 profiles::consul::server::acl_tokens_initial_management: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAi1UH7AZirJ1PdxWy+KEgS5ufm0wbn2xy9rkg14hKYpcVjBa4pOZpSLMGMiiUpBIqBytDMZM4ezYa/luktpkBImJbM/TE16beGtsacQGA+9eZk2Tihs9GR2qbAQiu5lLITiDlwNnf0GeWdqHM8CTeD68DczQF320d9U14/k6pG/7z+w/MGLcjsQoSuOFTm42JVn1BI46t1CYSCHMXQc/9Tfs+FzI+vumohI8DxAYBIuyzU5HBX/MntAsvD/yixMJS1pZL9WwgqZJC/wK34rVRB39DpxWf/WROrI+WLuSJwr7WBjaeF9Ju+89WKCgsI53EWhFTj8GgDZm/jqPoE478NjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAoACRzJdQKNYXZv6cghFIIgDAzB81DMcuY815nb8POtZpiA06jT/068AoZmSctHoFK/zW9tY229N5r1Tb+WHElqLk=]
 profiles::consul::server::acl_tokens_default: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAh4Ag95xgkIZHL0gP9OLnZauih0dB1/2l9Jzw8mP3OiIv7fw23otHYONlS3Emtj7oxW8MKcZGKDCzwCT6T2p+V5wx1n15wr2J+FmL24VbclJwrMPQ4AdgP359B9h21uoyo7Zdy7RuuvLfkU1fWXbs3SeWbi2HJs1Ed1/oI1jzr3OgwMbVtbyzd1VuAXeZ9bHQG3IA8z+w/k5m61th0HTyHjw7eldQulbohDuwv545z9axHEoHKCRT2a3ZwBufV2ST6Dm3g9GERzXE9Adp9DQC5adqM74wfsujOMLK2QFJSSIOj2uCs1CpEnrNrQ8zjP3fudM2z3l7KdSHZazEamCSxTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBY/Tn9tzEKYc5dxnzP2rP7gDBWKgVP3lf2T4Q0WPQt3ns0E6RUSO6OtBegb/5qDyohY2nsDeJTnMKOYzYt/J1PhnY=]
 profiles::consul::server::acl_tokens_replication: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAEzTIxbaAR/TZYnC671aBhiahsfwf3ieyeSHpD5hQm40sMEF/fXlEDijq9i2ykPikm94074j1Uo4HNzv/V9GTf0NSC/64t61jJ/Ya3QKa5f/36+DcRj3lcETsSIyhWwmU+E+zkY8I2r68MtXAuvSoMSMZdpgWSkPx/3FFgZlrsg//bzDu69jS9cx4UK582N3A6QN4Uy/qwYtJcm+2iTQlqqgGRWGqnSgTirxhegxPbJGWTDoAEpAL4/DyF5/hqcUn6mgoSfAsHF3loPHOqN30lG+9o0THWJ9B8Gf4W/1X2UWA/avmUnqBnumGoz0p7AYdNgpW+qLl2rk4lyGYa4kvQjBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBD9KSLH3Pn/1EIzgVJM+8VOgDDpKWzHfSVsfUMyTD0XIRPGclTdmxqCnsdhKNqmUfSjkYIf2nI9rQtSK6n42TYuO4k=]
-profiles::dns::updater::key_secret: ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAJ4GrN3W4zGfVygfji1MeGyAlmlFN45TVP5st6b66jSKeDYNg9BwdnXahUzkmKvFIYdPWyvXqJdvuyUNz7lWr1dw/pTNHrvqUKSvI0C5TOwmtrTwXtTvucDGs7c7FZDfgUxvLs/Vrp0ZdW3C81QXWNBI8vBfVZyiegPNq+5mX7mtcrFE7jZj1g1qwpg/30lrlCguPyXqnmdtBa7VAyU2Al1UZwkrKIYSAOb9GTnvM6bk5L26pW/NXGj+ffacwf58b4AiKCak5lhoCIInVnlogPuThxULN4RF+MYAVIlgwk159CR1abjWjioxvmP7XgjQUwUuTlgoLh9fDZa0amDeFZTBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBEb1BTjhuTN9wGrxc2rdQVgDCctNdt72UR/Hj4wteK7XIX6fF43+7FYKBZnPtyWIWalwDniH//H3gfsJ+ULqSb82w=]

hieradata/common.yaml

@@ -146,8 +146,6 @@ lookup_options:
       strategy: deep
   profiles::etcd::node::initial_cluster_token:
     convert_to: Sensitive
-  profiles::dns::updater::key_secret:
-    convert_to: Sensitive
   sysctl::base::values:
     merge:
       strategy: deep

hieradata/os/AlmaLinux/all_releases.yaml

@@ -78,13 +78,6 @@ profiles::yum::global::repos:
     gpgkey: https://git.query.consul/api/packages/unkinben/rpm/repository.key
     gpgcheck: false
     mirrorlist: absent
-  rpm-internal:
-    name: rpm-internal
-    descr: rpm-internal repository
-    target: /etc/yum.repos.d/rpm-internal.repo
-    baseurl: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/rpm-internal/
-    gpgcheck: false
-    mirrorlist: absent
 
   # Additional repositories - default to absent, roles can override with ensure: present
   # FRRouting repositories

hieradata/roles/infra/storage/vault.yaml

@@ -28,5 +28,3 @@ profiles::nginx::simpleproxy::proxy_path: '/'
 
 profiles::packages::include:
   openbao-plugins: {}
-  openbao-plugin-secrets-litellm: {}
-  openbao-plugin-secrets-gpg: {}

site/profiles/manifests/dns/updater.pp

@@ -23,12 +23,6 @@ class profiles::dns::updater (
   Stdlib::AbsolutePath        $state_dir       = '/var/lib/dns-updater',
   Stdlib::AbsolutePath        $config_dir      = '/etc/dns-updater',
   Stdlib::AbsolutePath        $master_basedir  = lookup('profiles::dns::master::basedir'),
-  # dns-updater daemon (replaces the dns-update shell script).
-  String                      $package_ensure   = 'installed',
-  Stdlib::AbsolutePath        $api_socket       = '/run/dns-updater/api.sock',
-  String                      $resync           = '10m',
-  Enum['debug', 'info', 'warn', 'error'] $log_level = 'info',
-  Boolean                     $watch_interfaces = true,
 ) {
 
   $state_file  = "${state_dir}/applied"
@@ -97,57 +91,36 @@ class profiles::dns::updater (
         })),
       }
 
-      # dns-updater daemon: watches the records file (inotify) and network
-      # interfaces, pushes TSIG-signed RFC2136 updates to $server natively.
-      package { 'dns-updater':
-        ensure => $package_ensure,
-      }
-
-      $env_content = @("ENV")
-        # Managed by puppet (profiles::dns::updater).
-        DNS_UPDATER_SERVER=${server}
-        DNS_UPDATER_KEY_FILE=${key_file}
-        DNS_UPDATER_RECORDS_FILE=${records_file}
-        DNS_UPDATER_STATE_FILE=${state_file}
-        DNS_UPDATER_API=${api_socket}
-        DNS_UPDATER_RESYNC=${resync}
-        DNS_UPDATER_WATCH_INTERFACES=${watch_interfaces}
-        DNS_UPDATER_LOG_LEVEL=${log_level}
-        | ENV
-
-      file { "${config_dir}/env":
+      file { '/usr/local/bin/dns-update':
         ensure  => file,
         owner   => 'root',
         group   => 'root',
-        mode    => '0644',
-        content => $env_content,
-        require => [File[$config_dir], Package['dns-updater']],
-      }
-
-      # Restart only on config/key change; records-file changes are picked up by
-      # the daemon's own inotify watch, so no service churn on record edits.
-      service { 'dns-updater':
-        ensure    => running,
-        enable    => true,
-        subscribe => [File["${config_dir}/env"], File[$key_file]],
-        require   => [Package['dns-updater'], Concat[$records_file], File[$key_file]],
-      }
-
-      # Retire the previous shell-based mechanism.
-      file { '/usr/local/bin/dns-update':
-        ensure => absent,
+        mode    => '0755',
+        content => epp('profiles/dns/dns-update.sh.epp', {
+            'server'       => $server,
+            'key_file'     => $key_file,
+            'records_file' => $records_file,
+            'state_file'   => $state_file,
+        }),
       }
 
       systemd::unit_file { 'dns-update.service':
-        ensure => absent,
-        active => false,
-        enable => false,
+        content => epp('profiles/dns/dns-update.service.epp', { 'script' => '/usr/local/bin/dns-update' }),
       }
 
+      # The .path unit watches the records file and triggers the service.
       systemd::unit_file { 'dns-update.path':
-        ensure => absent,
-        active => false,
-        enable => false,
+        content => epp('profiles/dns/dns-update.path.epp', { 'records_file' => $records_file }),
+        active  => true,
+        enable  => true,
+      }
+
+      # Also apply within the puppet run whenever the records change.
+      exec { 'dns-update-apply':
+        command     => '/usr/local/bin/dns-update',
+        refreshonly => true,
+        subscribe   => Concat[$records_file],
+        require     => [File['/usr/local/bin/dns-update'], File[$key_file]],
       }
     }
   }

site/profiles/templates/dns/dns-update.path.epp

@@ -0,0 +1,10 @@
+<%- | String $records_file | -%>
+[Unit]
+Description=Watch the DNS records file and apply changes
+
+[Path]
+PathModified=<%= $records_file %>
+Unit=dns-update.service
+
+[Install]
+WantedBy=multi-user.target

site/profiles/templates/dns/dns-update.service.epp

@@ -0,0 +1,9 @@
+<%- | String $script | -%>
+[Unit]
+Description=Apply host DNS records via nsupdate
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=<%= $script %>

site/profiles/templates/dns/dns-update.sh.epp

@@ -0,0 +1,56 @@
+<%- | String $server, String $key_file, String $records_file, String $state_file | -%>
+#!/bin/bash
+# Managed by puppet (profiles::dns::updater). Applies this host's records to the
+# authoritative DNS server via TSIG nsupdate. Only the delta since the last
+# successful run is sent; removed records are deleted.
+set -euo pipefail
+
+SERVER="<%= $server %>"
+KEYFILE="<%= $key_file %>"
+RECORDS="<%= $records_file %>"
+STATE="<%= $state_file %>"
+
+[ -f "$RECORDS" ] || exit 0
+touch "$STATE"
+
+# Format per line: zone|name|type|ttl|value  (name is relative to zone, or @).
+desired="$(grep -vE '^[[:space:]]*(#|$)' "$RECORDS" | sort -u || true)"
+applied="$(grep -vE '^[[:space:]]*(#|$)' "$STATE" 2>/dev/null | sort -u || true)"
+
+[ "$desired" = "$applied" ] && exit 0
+
+fqdn() { # name zone
+  if [ -z "$1" ] || [ "$1" = "@" ]; then printf '%s.' "$2"; else printf '%s.%s.' "$1" "$2"; fi
+}
+
+msg="$(mktemp)"
+trap 'rm -f "$msg"' EXIT
+printf 'server %s\n' "$SERVER" >> "$msg"
+
+# Process per zone so each UPDATE message targets a single zone.
+zones="$(printf '%s\n%s\n' "$desired" "$applied" | cut -d'|' -f1 | sort -u | grep -v '^$' || true)"
+for zone in $zones; do
+  printf 'zone %s.\n' "$zone" >> "$msg"
+  # Additions/updates: replace the RRset for every desired record in this zone.
+  printf '%s\n' "$desired" | awk -F'|' -v z="$zone" 'NF>=5 && $1==z' | \
+    while IFS='|' read -r z name type ttl value; do
+      f="$(fqdn "$name" "$z")"
+      printf 'update delete %s %s\n' "$f" "$type" >> "$msg"
+      printf 'update add %s %s %s %s\n' "$f" "$ttl" "$type" "$value" >> "$msg"
+    done
+  # Deletions: records present last run but gone now.
+  comm -23 <(printf '%s\n' "$applied") <(printf '%s\n' "$desired") | \
+    awk -F'|' -v z="$zone" 'NF>=5 && $1==z' | \
+    while IFS='|' read -r z name type ttl value; do
+      f="$(fqdn "$name" "$z")"
+      printf 'update delete %s %s %s\n' "$f" "$type" "$value" >> "$msg"
+    done
+  printf 'send\n' >> "$msg"
+done
+
+if nsupdate -k "$KEYFILE" "$msg"; then
+  printf '%s\n' "$desired" > "$STATE"
+else
+  echo "dns-update: nsupdate to ${SERVER} failed" >&2
+  exit 1
+fi