diff --git a/.woodpecker/backfill-test.yaml b/.woodpecker/backfill-test.yaml new file mode 100644 index 0000000..431f4dd --- /dev/null +++ b/.woodpecker/backfill-test.yaml @@ -0,0 +1,20 @@ +when: + - event: pull_request + +steps: + - name: test + image: golang:1.25 + directory: tools/backfill + commands: + - go vet ./... + - go test -race ./... + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 diff --git a/Makefile b/Makefile index fa3600f..78c0b70 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,12 @@ -.PHONY: init plan apply format pre-commit +.PHONY: init plan apply format pre-commit backfill + +# Hosts whose hardware reality is backfilled from pdbmux into NetBox. Override to +# add/limit hosts, e.g. `make backfill BACKFILL_HOSTS="prodnxsr0001 prodnxsr0002"`. +BACKFILL_HOSTS ?= prodnxsr0001 prodnxsr0002 prodnxsr0003 prodnxsr0004 prodnxsr0005 \ + prodnxsr0006 prodnxsr0007 prodnxsr0008 prodnxsr0009 prodnxsr0010 prodnxsr0011 \ + prodnxsr0012 prodnxsr0013 +PDBMUX_URL ?= https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts +BACKFILL_OUT ?= config/au/syd1/reality VAULT_AUTH_METHOD ?= approle VAULT_K8S_ROLE ?= woodpecker_terraform_infra @@ -35,3 +43,9 @@ format: pre-commit: @uvx pre-commit run --all-files + +# Regenerate NetBox reality YAML from pdbmux. Idempotent with stable ordering, +# so re-running against unchanged facts leaves a clean git diff. Needs network +# reachability to pdbmux (its HTTPS gateway is reachable from CI/workstations). +backfill: + @go -C tools/backfill run . --url "$(PDBMUX_URL)" --out "$(CURDIR)/$(BACKFILL_OUT)" $(BACKFILL_HOSTS) diff --git a/README.md b/README.md index 293d72f..e63df39 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ intent and lets discovery/PuppetDB fill hardware reality. config///subnets/.yaml # one file per subnet (prefix + DHCP scope) config///networks/.yaml # logical networks devices join config///devices/.yaml # one file per device (intent only) +config///reality/.yaml # one file per device (generated reality) config///managed-ips.yaml # extra manually-managed IPs environments///terragrunt.hcl # one Terragrunt env per region/dc modules/infra/ # the module -ci/puppetdb_backfill.py # reality backfill generator (local/in-cluster) +tools/backfill/ # pdbmux -> reality YAML generator (Go) ``` `make plan` / `make apply` reconcile the YAML against NetBox + Kea. State lives in @@ -78,13 +79,37 @@ Mapping: `netbox_device` (+ `netbox_manufacturer`/`netbox_device_type`/ so bootapi can key the PXE boot on MAC until the discovery image exists — remove it once discovery/backfill populates the real NICs. -## PuppetDB reality backfill +## Reality backfill (`reality/.yaml`) -`ci/puppetdb_backfill.py` sweeps already-provisioned hosts and emits their hardware -reality (serial/model/UUID + every interface's MAC/IPs, incl. overlay/loopback -addresses Cobbler lacks) as reviewable YAML — the NetBox *reality* side, not the -intent above. PuppetDB needs no auth but is **not reachable from CI**; run it -in-cluster/from puppet infra. Wiring its output into NetBox is a follow-up (issue). +`tools/backfill` (Go) queries **pdbmux** — the PuppetDB multiplexer at +`https://pdbmux.k8s.syd1.au.unkin.net`, which merges the old (Consul) and new +(k8s) PuppetDBs and needs no auth — for already-provisioned hosts and emits their +hardware reality as reviewable per-host YAML: + +- **identity** — serial, model, UUID (from `dmi.product`); +- **interfaces** — every recordable NIC with MAC + CIDR IPs, including the + overlay/loopback/kube-lb addresses Cobbler never had. Real NICs are always + kept; virtual interfaces are kept only when they carry a routable address, + which drops ephemeral Calico veths and `lo` while keeping `flannel.1`, + `kube-lb0`, `loopback*`. Interface names are taken from Facter, never assumed; +- **inventory** — CPU model/count, total RAM bytes, and physical disks (Ceph RBD + and other virtual block devices are skipped). + +```sh +make backfill # all 13 existing physicals +make backfill BACKFILL_HOSTS="prodnxsr0001" # a subset +``` + +Output is deterministic and idempotent (stable ordering, sorted lists), so +re-running against unchanged facts leaves a clean diff. Unlike raw PuppetDB, +pdbmux's HTTPS gateway is reachable from CI/workstations. The generator only +writes committed config; `netbox_device_interface`/`netbox_mac_address`/ +`netbox_ip_address`/`netbox_inventory_item` (plus device `serial`) reconcile it +into NetBox on the next apply. NetBox stays authoritative for IP data. + +Reality is wired into NetBox only for hosts that also have an intent +`devices/.yaml`; the 13 existing physicals get their intent seeded +separately, so until then their `reality/*.yaml` is committed but inert. ## Providers diff --git a/ci/puppetdb_backfill.py b/ci/puppetdb_backfill.py deleted file mode 100755 index f67e9d0..0000000 --- a/ci/puppetdb_backfill.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -"""Backfill NetBox device *reality* from PuppetDB for already-provisioned hosts. - -This emits the hardware-owned facts a running host reports (serial, model, UUID, -and every interface's name/MAC/IPs including the overlay/loopback addresses -Cobbler never had) as reviewable per-host YAML. It targets the *reality* side of -NetBox (interfaces + inventory) for the ~13 existing physicals; it is NOT the -intent `config/.../devices/*.yaml` (a human writes those). - -PuppetDB needs no auth (TLS terminates at the gateway; upstream is plain HTTP). -It is NOT reachable from CI, so run this in-cluster or from puppet infra: - - ./puppetdb_backfill.py --out ../reality \ - --url http://puppetdb.puppet.svc.cluster.local:8080/pdb/query/v4/facts \ - prodnxsr0001 prodnxsr0002 ... - -Fact paths follow Facter conventions (networking.*, dmi.*); verify against a live -factset (`GET /pdb/query/v4/factsets`) before trusting output on a new estate. -""" -import argparse -import json -import os -import sys -import urllib.parse -import urllib.request - -DEFAULT_URL = "http://puppetdb.puppet.svc.cluster.local:8080/pdb/query/v4/facts" - - -def query_facts(url, certname): - ast = json.dumps(["=", "certname", certname]) - full = url + "?" + urllib.parse.urlencode({"query": ast}) - with urllib.request.urlopen(full, timeout=30) as resp: - rows = json.load(resp) - return {row["name"]: row["value"] for row in rows} - - -def device_reality(certname, facts): - networking = facts.get("networking", {}) or {} - ifaces = networking.get("interfaces", {}) or {} - dmi = facts.get("dmi", {}) or {} - product = dmi.get("product", {}) or {} - - interfaces = [] - for name, data in sorted(ifaces.items()): - if name == "lo": - continue - ips = [] - for b in (data.get("bindings", []) or []): - if b.get("address"): - ips.append(b["address"]) - if data.get("ip") and data["ip"] not in ips: - ips.append(data["ip"]) - interfaces.append({ - "name": name, - "mac": data.get("mac"), - "ips": ips, - }) - - return { - "device": certname.split(".")[0], - "serial": product.get("serial_number") or facts.get("serialnumber"), - "model": product.get("name") or facts.get("productname"), - "uuid": product.get("uuid") or facts.get("uuid"), - "interfaces": interfaces, - } - - -def to_yaml(d): - # tiny dependency-free YAML emitter for this fixed shape - out = [ - f"device: {d['device']}", - f"serial: {d['serial'] or ''}", - f"model: {d['model'] or ''}", - f"uuid: {d['uuid'] or ''}", - "interfaces:", - ] - for i in d["interfaces"]: - out.append(f" - name: {i['name']}") - out.append(f" mac: {i['mac'] or ''}") - out.append(" ips: [%s]" % ", ".join(i["ips"])) - return "\n".join(out) + "\n" - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("hosts", nargs="+", help="certnames or short hostnames") - ap.add_argument("--url", default=os.environ.get("PUPPETDB_URL", DEFAULT_URL)) - ap.add_argument("--out", help="write .yaml here instead of stdout") - args = ap.parse_args() - - for host in args.hosts: - certname = host if "." in host else f"{host}.main.unkin.net" - try: - facts = query_facts(args.url, certname) - except Exception as e: # noqa: BLE001 - print(f"# {certname}: query failed: {e}", file=sys.stderr) - continue - if not facts: - print(f"# {certname}: no facts (not in PuppetDB)", file=sys.stderr) - continue - text = to_yaml(device_reality(certname, facts)) - if args.out: - os.makedirs(args.out, exist_ok=True) - path = os.path.join(args.out, f"{certname.split('.')[0]}.yaml") - with open(path, "w") as fh: - fh.write(text) - print(f"wrote {path}", file=sys.stderr) - else: - print(text) - - -if __name__ == "__main__": - main() diff --git a/config/au/syd1/reality/prodnxsr0001.yaml b/config/au/syd1/reality/prodnxsr0001.yaml new file mode 100644 index 0000000..164d567 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0001.yaml @@ -0,0 +1,73 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0001.main.unkin.net +device: "prodnxsr0001" +serial: "4RW6QM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0052-5710-8036-b4c04f514d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:75:c3:60" + mtu: 1500 + physical: true + ips: + - "198.18.15.1/24" + - "198.18.19.1/32" + - name: "enp3s0" + mac: "00:ac:d0:00:00:50" + mtu: 1500 + physical: true + ips: + - "198.18.19.1/32" + - "198.18.21.1/24" + - name: "flannel.1" + mac: "56:ac:21:3e:cd:9b" + mtu: 1450 + physical: false + ips: + - "10.42.0.0/32" + - name: "kube-lb0" + mac: "1e:4f:62:0c:e8:e1" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "f2:3d:ef:64:42:51" + mtu: 1500 + physical: false + ips: + - "198.18.19.1/32" + - name: "loopback1" + mac: "aa:36:f9:d5:f0:1d" + mtu: 1500 + physical: false + ips: + - "198.18.22.1/32" + - name: "loopback2" + mac: "96:aa:ea:a3:6d:c5" + mtu: 1500 + physical: false + ips: + - "198.18.23.1/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33260900352 + disks: + - name: "nvme0n1" + model: "IM2P33F8-256GD1" + serial: "2M3029AD7KFE" + size_bytes: 256060514304 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC651303WB1P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0002.yaml b/config/au/syd1/reality/prodnxsr0002.yaml new file mode 100644 index 0000000..eeda70f --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0002.yaml @@ -0,0 +1,73 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0002.main.unkin.net +device: "prodnxsr0002" +serial: "4TC5QM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0054-4310-8035-b4c04f514d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:74:b6:08" + mtu: 1500 + physical: true + ips: + - "198.18.15.2/24" + - "198.18.19.2/32" + - name: "enp3s0" + mac: "00:e0:4c:68:08:43" + mtu: 1500 + physical: true + ips: + - "198.18.19.2/32" + - "198.18.21.2/24" + - name: "flannel.1" + mac: "02:af:20:b3:b1:5e" + mtu: 1450 + physical: false + ips: + - "10.42.1.0/32" + - name: "kube-lb0" + mac: "6e:0e:5f:ed:80:48" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "36:7a:28:ec:2a:66" + mtu: 1500 + physical: false + ips: + - "198.18.19.2/32" + - name: "loopback1" + mac: "82:3e:3b:b2:ea:14" + mtu: 1500 + physical: false + ips: + - "198.18.22.2/32" + - name: "loopback2" + mac: "42:e0:d5:e8:c1:72" + mtu: 1500 + physical: false + ips: + - "198.18.23.2/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261576192 + disks: + - name: "nvme0n1" + model: "SAMSUNG MZVL2256HCHQ-00BL7" + serial: "S64MNF0RB00821" + size_bytes: 256060514304 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC65100A5W1P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0003.yaml b/config/au/syd1/reality/prodnxsr0003.yaml new file mode 100644 index 0000000..dfa6b5d --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0003.yaml @@ -0,0 +1,73 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0003.main.unkin.net +device: "prodnxsr0003" +serial: "1PN9BS2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0050-4e10-8039-b1c04f425332" +interfaces: + - name: "enp2s0" + mac: "b8:85:84:a3:25:c5" + mtu: 1500 + physical: true + ips: + - "198.18.15.3/24" + - "198.18.19.3/32" + - name: "enp3s0" + mac: "00:e0:4c:68:07:82" + mtu: 1500 + physical: true + ips: + - "198.18.19.3/32" + - "198.18.21.3/24" + - name: "flannel.1" + mac: "6a:66:4f:b8:fb:f5" + mtu: 1450 + physical: false + ips: + - "10.42.2.0/32" + - name: "kube-lb0" + mac: "3e:4c:cd:3c:33:ec" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "6a:a4:6f:c2:29:6e" + mtu: 1500 + physical: false + ips: + - "198.18.19.3/32" + - name: "loopback1" + mac: "4e:4a:46:75:4e:5a" + mtu: 1500 + physical: false + ips: + - "198.18.22.3/32" + - name: "loopback2" + mac: "be:1d:95:db:e1:ef" + mtu: 1500 + physical: false + ips: + - "198.18.23.3/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261572096 + disks: + - name: "nvme0n1" + model: "WDC PC SN520 SDAPNUW-512G-1006" + serial: "191342472105" + size_bytes: 512110190592 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC65100A7R1P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0004.yaml b/config/au/syd1/reality/prodnxsr0004.yaml new file mode 100644 index 0000000..1634de6 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0004.yaml @@ -0,0 +1,75 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0004.main.unkin.net +device: "prodnxsr0004" +serial: "4RMYPM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0052-4d10-8059-b4c04f504d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:75:d5:00" + mtu: 1500 + physical: true + ips: + - "198.18.15.4/24" + - "198.18.19.4/32" + - name: "enp3s0" + mac: "00:ac:d0:00:00:43" + mtu: 1500 + physical: true + ips: + - "198.18.19.4/32" + - "198.18.21.4/24" + - name: "flannel.1" + mac: "b6:c2:ee:87:d6:1b" + mtu: 1450 + physical: false + ips: + - "10.42.4.0/32" + - name: "kube-lb0" + mac: "6e:ae:97:31:4c:56" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.6/32" + - "198.18.200.8/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "32:4f:54:04:f8:da" + mtu: 1500 + physical: false + ips: + - "198.18.19.4/32" + - name: "loopback1" + mac: "56:da:d9:21:48:5f" + mtu: 1500 + physical: false + ips: + - "198.18.22.4/32" + - name: "loopback2" + mac: "72:7e:58:5b:1e:d6" + mtu: 1500 + physical: false + ips: + - "198.18.23.4/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261576192 + disks: + - name: "nvme0n1" + model: "SAMSUNG MZVLQ256HBJD-00BH1" + serial: "S672NX0T347235" + size_bytes: 256060514304 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC65100A8G1P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0005.yaml b/config/au/syd1/reality/prodnxsr0005.yaml new file mode 100644 index 0000000..f2c895b --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0005.yaml @@ -0,0 +1,74 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0005.main.unkin.net +device: "prodnxsr0005" +serial: "5TQW8T2" +model: "OptiPlex 3060" +uuid: "4c4c4544-0054-5110-8057-b5c04f385432" +interfaces: + - name: "enp1s0" + mac: "54:bf:64:a0:08:64" + mtu: 1500 + physical: true + ips: + - "198.18.15.5/24" + - "198.18.19.5/32" + - name: "enp3s0" + mac: "00:e0:4c:68:07:79" + mtu: 1500 + physical: true + ips: + - "198.18.19.5/32" + - "198.18.21.5/24" + - name: "flannel.1" + mac: "02:2e:f5:c7:79:05" + mtu: 1450 + physical: false + ips: + - "10.42.3.0/32" + - name: "kube-lb0" + mac: "6a:78:43:e0:84:b2" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.7/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "fa:85:3b:4d:33:9f" + mtu: 1500 + physical: false + ips: + - "198.18.19.5/32" + - name: "loopback1" + mac: "aa:cc:25:07:24:11" + mtu: 1500 + physical: false + ips: + - "198.18.22.5/32" + - name: "loopback2" + mac: "fa:d6:a7:27:3c:e6" + mtu: 1500 + physical: false + ips: + - "198.18.23.5/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-8400T CPU @ 1.70GHz" + cores: 6 + count: 6 + memory_bytes: 33197555712 + disks: + - name: "nvme0n1" + model: "SAMSUNG MZVLQ256HBJD-00BH1" + serial: "S672NS2T518945" + size_bytes: 256060514304 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC65100A891P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0006.yaml b/config/au/syd1/reality/prodnxsr0006.yaml new file mode 100644 index 0000000..285bd6e --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0006.yaml @@ -0,0 +1,76 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0006.main.unkin.net +device: "prodnxsr0006" +serial: "4RWYPM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0052-5710-8059-b4c04f504d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:75:10:8d" + mtu: 1500 + physical: true + ips: + - "198.18.15.6/24" + - "198.18.19.6/32" + - name: "enp3s0" + mac: "00:ac:d0:00:00:53" + mtu: 1500 + physical: true + ips: + - "198.18.19.6/32" + - "198.18.21.6/24" + - name: "flannel.1" + mac: "86:9e:22:f3:f2:9f" + mtu: 1450 + physical: false + ips: + - "10.42.7.0/32" + - name: "kube-lb0" + mac: "5a:60:5a:b8:61:3b" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.6/32" + - "198.18.200.7/32" + - "198.18.200.8/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "36:79:13:ec:24:be" + mtu: 1500 + physical: false + ips: + - "198.18.19.6/32" + - name: "loopback1" + mac: "4e:da:32:4f:1a:5b" + mtu: 1500 + physical: false + ips: + - "198.18.22.6/32" + - name: "loopback2" + mac: "5e:7d:66:a9:50:74" + mtu: 1500 + physical: false + ips: + - "198.18.23.6/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261568000 + disks: + - name: "nvme0n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7687513575" + size_bytes: 1024209543168 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC65100A771P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0007.yaml b/config/au/syd1/reality/prodnxsr0007.yaml new file mode 100644 index 0000000..f353c08 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0007.yaml @@ -0,0 +1,76 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0007.main.unkin.net +device: "prodnxsr0007" +serial: "4SH4QM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0053-4810-8034-b4c04f514d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:74:b4:27" + mtu: 1500 + physical: true + ips: + - "198.18.15.7/24" + - "198.18.19.7/32" + - name: "enp3s0" + mac: "00:ac:d0:00:00:5b" + mtu: 1500 + physical: true + ips: + - "198.18.19.7/32" + - "198.18.21.7/24" + - name: "flannel.1" + mac: "12:a8:e5:df:a8:11" + mtu: 1450 + physical: false + ips: + - "10.42.5.0/32" + - name: "kube-lb0" + mac: "56:f2:92:e9:e3:ff" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.199.53/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.6/32" + - "198.18.200.7/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "9e:d4:1e:4f:77:78" + mtu: 1500 + physical: false + ips: + - "198.18.19.7/32" + - name: "loopback1" + mac: "de:7c:95:3a:84:7d" + mtu: 1500 + physical: false + ips: + - "198.18.22.7/32" + - name: "loopback2" + mac: "96:5c:4a:57:6b:3b" + mtu: 1500 + physical: false + ips: + - "198.18.23.7/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261572096 + disks: + - name: "nvme0n1" + model: "SAMSUNG MZVLQ256HBJD-00BH1" + serial: "S672NS2T521338" + size_bytes: 256060514304 + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC651407V41P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0008.yaml b/config/au/syd1/reality/prodnxsr0008.yaml new file mode 100644 index 0000000..0b4b946 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0008.yaml @@ -0,0 +1,73 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0008.main.unkin.net +device: "prodnxsr0008" +serial: "4T34QM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0054-3310-8034-b4c04f514d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:75:06:18" + mtu: 1500 + physical: true + ips: + - "198.18.15.8/24" + - "198.18.19.8/32" + - name: "enp3s0" + mac: "00:e0:4c:68:08:4b" + mtu: 1500 + physical: true + ips: + - "198.18.19.8/32" + - "198.18.21.8/24" + - name: "flannel.1" + mac: "0e:58:18:dc:77:da" + mtu: 1450 + physical: false + ips: + - "10.42.6.0/32" + - name: "kube-lb0" + mac: "e6:48:68:86:a6:70" + mtu: 1500 + physical: false + ips: + - "198.18.199.0/32" + - "198.18.200.0/32" + - "198.18.200.1/32" + - "198.18.200.2/32" + - "198.18.200.3/32" + - "198.18.200.4/32" + - "198.18.200.5/32" + - "198.18.200.9/32" + - name: "loopback0" + mac: "26:6f:96:df:29:58" + mtu: 1500 + physical: false + ips: + - "198.18.19.8/32" + - name: "loopback1" + mac: "2e:30:dc:43:e6:c4" + mtu: 1500 + physical: false + ips: + - "198.18.22.8/32" + - name: "loopback2" + mac: "de:ee:01:46:10:4b" + mtu: 1500 + physical: false + ips: + - "198.18.23.8/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33261576192 + disks: + - name: "nvme0n1" + model: "SAMSUNG MZVLW256HEHP-000H7" + serial: "S365NX0K803338" + size_bytes: 256060514304 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC635301YE1P2OGN" + size_bytes: 1200243695616 diff --git a/config/au/syd1/reality/prodnxsr0009.yaml b/config/au/syd1/reality/prodnxsr0009.yaml new file mode 100644 index 0000000..ee277c7 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0009.yaml @@ -0,0 +1,80 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0009.main.unkin.net +device: "prodnxsr0009" +serial: "DXN4X63" +model: "OptiPlex 7080" +uuid: "4c4c4544-0058-4e10-8034-c4c04f583633" +interfaces: + - name: "brcom1" + mac: "00:16:3e:05:2d:0a" + mtu: 1500 + physical: false + ips: + - "198.18.19.9/32" + - "198.18.25.254/24" + - "fd42:b65e:b59e:de1c::1/64" + - name: "brdmz1" + mac: "00:16:3e:8d:17:aa" + mtu: 1500 + physical: false + ips: + - "198.18.19.9/32" + - "198.18.24.14/28" + - name: "brwan1" + mac: "00:16:3e:49:18:62" + mtu: 1500 + physical: false + ips: + - "198.18.20.14/28" + - "fd42:f06c:628:43ae::1/64" + - name: "enp2s0" + mac: "70:b5:e8:38:e9:8d" + mtu: 1500 + physical: true + ips: + - "198.18.15.9/24" + - "198.18.19.9/32" + - name: "enp3s0" + mac: "00:e0:4c:68:0f:5d" + mtu: 1500 + physical: true + ips: + - "198.18.19.9/32" + - "198.18.21.9/24" + - name: "loopback0" + mac: "72:b5:e6:93:78:08" + mtu: 1500 + physical: false + ips: + - "198.18.19.9/32" + - name: "loopback1" + mac: "ea:0d:ae:2d:36:82" + mtu: 1500 + physical: false + ips: + - "198.18.22.9/32" + - name: "loopback2" + mac: "ce:a4:8a:c5:cb:0e" + mtu: 1500 + physical: false + ips: + - "198.18.23.9/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz" + cores: 8 + count: 16 + memory_bytes: 66850242560 + disks: + - name: "nvme0n1" + model: "KINGSTON SNV3S500G" + serial: "50026B72835F70DC" + size_bytes: 500107862016 + - name: "nvme1n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7686FC8586" + size_bytes: 1024209543168 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC712010Q91P6PGN" + size_bytes: 1600321314816 diff --git a/config/au/syd1/reality/prodnxsr0010.yaml b/config/au/syd1/reality/prodnxsr0010.yaml new file mode 100644 index 0000000..1968438 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0010.yaml @@ -0,0 +1,80 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0010.main.unkin.net +device: "prodnxsr0010" +serial: "DXL3X63" +model: "OptiPlex 7080" +uuid: "4c4c4544-0058-4c10-8033-c4c04f583633" +interfaces: + - name: "brcom1" + mac: "00:16:3e:48:7a:41" + mtu: 1500 + physical: false + ips: + - "198.18.19.10/32" + - "198.18.26.254/24" + - "fd42:d01f:2449:bc96::1/64" + - name: "brdmz1" + mac: "00:16:3e:9d:d5:d1" + mtu: 1500 + physical: false + ips: + - "198.18.19.10/32" + - "198.18.24.30/28" + - name: "brwan1" + mac: "00:16:3e:82:39:bd" + mtu: 1500 + physical: false + ips: + - "198.18.20.30/28" + - "fd42:d67b:303f:6e24::1/64" + - name: "enp2s0" + mac: "70:b5:e8:38:e9:37" + mtu: 1500 + physical: true + ips: + - "198.18.15.10/24" + - "198.18.19.10/32" + - name: "enp3s0" + mac: "00:e0:4c:68:0f:de" + mtu: 1500 + physical: true + ips: + - "198.18.19.10/32" + - "198.18.21.10/24" + - name: "loopback0" + mac: "12:66:f8:64:7f:6d" + mtu: 1500 + physical: false + ips: + - "198.18.19.10/32" + - name: "loopback1" + mac: "9e:1c:e1:f0:df:2b" + mtu: 1500 + physical: false + ips: + - "198.18.22.10/32" + - name: "loopback2" + mac: "8e:96:81:e6:60:f3" + mtu: 1500 + physical: false + ips: + - "198.18.23.10/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz" + cores: 8 + count: 16 + memory_bytes: 66850258944 + disks: + - name: "nvme0n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7686FC8568" + size_bytes: 1024209543168 + - name: "nvme1n1" + model: "KINGSTON SNV3S500G" + serial: "50026B72835F709B" + size_bytes: 500107862016 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC71200Y391P6PGN" + size_bytes: 1600321314816 diff --git a/config/au/syd1/reality/prodnxsr0011.yaml b/config/au/syd1/reality/prodnxsr0011.yaml new file mode 100644 index 0000000..a43a438 --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0011.yaml @@ -0,0 +1,80 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0011.main.unkin.net +device: "prodnxsr0011" +serial: "DXL7X63" +model: "OptiPlex 7080" +uuid: "4c4c4544-0058-4c10-8037-c4c04f583633" +interfaces: + - name: "brcom1" + mac: "00:16:3e:f5:75:19" + mtu: 1500 + physical: false + ips: + - "198.18.19.11/32" + - "198.18.27.254/24" + - "fd42:8521:2898:632d::1/64" + - name: "brdmz1" + mac: "00:16:3e:51:57:10" + mtu: 1500 + physical: false + ips: + - "198.18.19.11/32" + - "198.18.24.46/28" + - name: "brwan1" + mac: "00:16:3e:89:a5:81" + mtu: 1500 + physical: false + ips: + - "198.18.20.46/28" + - "fd42:2822:1126:7d77::1/64" + - name: "enp2s0" + mac: "70:b5:e8:38:e9:0f" + mtu: 1500 + physical: true + ips: + - "198.18.15.11/24" + - "198.18.19.11/32" + - name: "enp3s0" + mac: "00:e0:4c:68:0f:55" + mtu: 1500 + physical: true + ips: + - "198.18.19.11/32" + - "198.18.21.11/24" + - name: "loopback0" + mac: "4a:e7:7c:fa:c4:08" + mtu: 1500 + physical: false + ips: + - "198.18.19.11/32" + - name: "loopback1" + mac: "8e:00:c7:54:02:55" + mtu: 1500 + physical: false + ips: + - "198.18.22.11/32" + - name: "loopback2" + mac: "4a:0a:b8:df:08:52" + mtu: 1500 + physical: false + ips: + - "198.18.23.11/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz" + cores: 8 + count: 16 + memory_bytes: 66850238464 + disks: + - name: "nvme0n1" + model: "KINGSTON SNV3S500G" + serial: "50026B72835F70FA" + size_bytes: 500107862016 + - name: "nvme1n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7686FC855C" + size_bytes: 1024209543168 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC712010EL1P6PGN" + size_bytes: 1600321314816 diff --git a/config/au/syd1/reality/prodnxsr0012.yaml b/config/au/syd1/reality/prodnxsr0012.yaml new file mode 100644 index 0000000..a85deac --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0012.yaml @@ -0,0 +1,80 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0012.main.unkin.net +device: "prodnxsr0012" +serial: "G0W0TC3" +model: "OptiPlex 7080" +uuid: "4c4c4544-0030-5710-8030-c7c04f544333" +interfaces: + - name: "brcom1" + mac: "00:16:3e:a6:72:58" + mtu: 1500 + physical: false + ips: + - "198.18.19.12/32" + - "198.18.28.254/24" + - "fd42:b712:24e:9015::1/64" + - name: "brdmz1" + mac: "00:16:3e:0c:d4:5b" + mtu: 1500 + physical: false + ips: + - "198.18.19.12/32" + - "198.18.24.62/28" + - name: "brwan1" + mac: "00:16:3e:66:a9:2d" + mtu: 1500 + physical: false + ips: + - "198.18.20.62/28" + - "fd42:c419:fbd5:1220::1/64" + - name: "enp2s0" + mac: "70:b5:e8:4f:05:1e" + mtu: 1500 + physical: true + ips: + - "198.18.15.12/24" + - "198.18.19.12/32" + - name: "enp3s0" + mac: "00:e0:4c:68:0f:e5" + mtu: 1500 + physical: true + ips: + - "198.18.19.12/32" + - "198.18.21.12/24" + - name: "loopback0" + mac: "32:11:8f:a0:3a:d7" + mtu: 1500 + physical: false + ips: + - "198.18.19.12/32" + - name: "loopback1" + mac: "12:d2:9c:88:ea:58" + mtu: 1500 + physical: false + ips: + - "198.18.22.12/32" + - name: "loopback2" + mac: "ca:f7:1c:b4:90:b3" + mtu: 1500 + physical: false + ips: + - "198.18.23.12/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz" + cores: 8 + count: 16 + memory_bytes: 66850258944 + disks: + - name: "nvme0n1" + model: "KINGSTON SNV3S500G" + serial: "50026B72835F70AE" + size_bytes: 500107862016 + - name: "nvme1n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7686FC855A" + size_bytes: 1024209543168 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC712010TL1P6PGN" + size_bytes: 1600321314816 diff --git a/config/au/syd1/reality/prodnxsr0013.yaml b/config/au/syd1/reality/prodnxsr0013.yaml new file mode 100644 index 0000000..956a62d --- /dev/null +++ b/config/au/syd1/reality/prodnxsr0013.yaml @@ -0,0 +1,80 @@ +# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts certname=prodnxsr0013.main.unkin.net +device: "prodnxsr0013" +serial: "H0W0TC3" +model: "OptiPlex 7080" +uuid: "4c4c4544-0030-5710-8030-c8c04f544333" +interfaces: + - name: "brcom1" + mac: "00:16:3e:7a:fd:76" + mtu: 1500 + physical: false + ips: + - "198.18.19.13/32" + - "198.18.29.254/24" + - "fd42:5b9:111b:8e6f::1/64" + - name: "brdmz1" + mac: "00:16:3e:47:0c:64" + mtu: 1500 + physical: false + ips: + - "198.18.19.13/32" + - "198.18.24.78/28" + - name: "brwan1" + mac: "00:16:3e:e9:7a:12" + mtu: 1500 + physical: false + ips: + - "198.18.20.78/28" + - "fd42:1492:b7fd:fced::1/64" + - name: "enp2s0" + mac: "70:b5:e8:4f:04:b0" + mtu: 1500 + physical: true + ips: + - "198.18.15.13/24" + - "198.18.19.13/32" + - name: "enp3s0" + mac: "00:e0:4c:68:0f:36" + mtu: 1500 + physical: true + ips: + - "198.18.19.13/32" + - "198.18.21.13/24" + - name: "loopback0" + mac: "0a:f5:e7:20:2e:5c" + mtu: 1500 + physical: false + ips: + - "198.18.19.13/32" + - name: "loopback1" + mac: "22:87:17:9e:32:db" + mtu: 1500 + physical: false + ips: + - "198.18.22.13/32" + - name: "loopback2" + mac: "b6:87:cc:0e:4e:83" + mtu: 1500 + physical: false + ips: + - "198.18.23.13/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz" + cores: 8 + count: 16 + memory_bytes: 66850271232 + disks: + - name: "nvme0n1" + model: "KINGSTON SNV3S500G" + serial: "50026B72835F70C5" + size_bytes: 500107862016 + - name: "nvme1n1" + model: "KINGSTON SKC3000S1024G" + serial: "50026B7686FC8569" + size_bytes: 1024209543168 + - name: "sda" + model: "INTEL SSDSC2BX01" + serial: "BTHC712010691P6PGN" + size_bytes: 1600321314816 diff --git a/environments/au/syd1/terragrunt.hcl b/environments/au/syd1/terragrunt.hcl index 9977bd2..3eab38c 100644 --- a/environments/au/syd1/terragrunt.hcl +++ b/environments/au/syd1/terragrunt.hcl @@ -29,6 +29,12 @@ locals { managed_ips_file = "${local.cfg_dir}/managed-ips.yaml" managed_ips = fileexists(local.managed_ips_file) ? yamldecode(file(local.managed_ips_file)) : [] + + reality_files = fileset(local.cfg_dir, "reality/*.yaml") + reality = { + for f in local.reality_files : + trimsuffix(basename(f), ".yaml") => yamldecode(file("${local.cfg_dir}/${f}")) + } } terraform { @@ -40,4 +46,5 @@ inputs = { networks = local.networks devices = local.devices managed_ips = local.managed_ips + reality = local.reality } diff --git a/modules/infra/devices.tf b/modules/infra/devices.tf index 8b6ade0..9f7bef6 100644 --- a/modules/infra/devices.tf +++ b/modules/infra/devices.tf @@ -54,6 +54,9 @@ resource "netbox_device" "this" { role_id = netbox_device_role.this[each.value.role].id site_id = tonumber(data.netbox_site.this[each.value.site].id) status = each.value.pxe ? "staged" : "active" + + # Serial is hardware reality; take it from the backfill when present. + serial = try(var.reality[each.key].serial, null) } # Transitional: bootapi keys PXE on MAC and no discovery image exists yet, so seed a diff --git a/modules/infra/reality.tf b/modules/infra/reality.tf new file mode 100644 index 0000000..db732b9 --- /dev/null +++ b/modules/infra/reality.tf @@ -0,0 +1,100 @@ +locals { + # Reality is wired only for hosts that also carry an intent device (the 13 + # existing physicals get their intent seeded in issue #2). Until then a host's + # reality/*.yaml is reviewed and committed but produces no NetBox resources. + reality_devices = { for k, r in var.reality : k => r if contains(keys(var.devices), k) } + + # ":" -> real interface. Names come from Facter, never assumed. + reality_interfaces = merge([ + for dname, r in local.reality_devices : { + for i in coalesce(r.interfaces, []) : "${dname}:${i.name}" => { + device = dname + name = i.name + mac = i.mac + mtu = i.mtu + physical = coalesce(i.physical, false) + } + } + ]...) + + reality_macs = { for k, i in local.reality_interfaces : k => i if i.mac != null && i.mac != "" } + + # "::" -> address on that interface (CIDR form). + reality_ips = merge([ + for dname, r in local.reality_devices : { + for pair in flatten([ + for i in coalesce(r.interfaces, []) : [ + for ip in coalesce(i.ips, []) : { device = dname, iface = i.name, ip = ip } + ] + ]) : "${pair.device}:${pair.iface}:${pair.ip}" => pair + } + ]...) + + # ":" -> one NetBox inventory item (CPU, memory, per-disk). + reality_inventory = merge([ + for dname, r in local.reality_devices : { + for item in concat( + try(r.inventory.cpu, null) == null ? [] : [{ + key = "cpu" + name = "CPU" + description = trimspace(format("%s x%d", try(r.inventory.cpu.model, ""), try(r.inventory.cpu.count, 0))) + part_id = try(r.inventory.cpu.model, null) + serial = null + }], + try(r.inventory.memory_bytes, null) == null ? [] : [{ + key = "memory" + name = "Memory" + description = format("%d bytes", r.inventory.memory_bytes) + part_id = null + serial = null + }], + [for d in try(r.inventory.disks, []) : { + key = "disk-${d.name}" + name = "Disk ${d.name}" + description = trimspace(format("%s %d bytes", try(d.model, ""), try(d.size_bytes, 0))) + part_id = try(d.model, null) + serial = try(d.serial, null) + }] + ) : "${dname}:${item.key}" => merge(item, { device = dname }) + } + ]...) +} + +# Real NICs and virtual interfaces (overlay/loopback/kube-lb) the host reports. +resource "netbox_device_interface" "reality" { + for_each = local.reality_interfaces + + device_id = netbox_device.this[each.value.device].id + name = each.value.name + type = each.value.physical ? "1000base-t" : "virtual" + mtu = each.value.mtu +} + +resource "netbox_mac_address" "reality" { + for_each = local.reality_macs + + mac_address = each.value.mac + device_interface_id = netbox_device_interface.reality[each.key].id +} + +# NetBox stays IP-authoritative; these record the addresses (incl. overlay / +# loopback / kube-lb VIPs) a running host actually carries. +resource "netbox_ip_address" "reality" { + for_each = local.reality_ips + + ip_address = each.value.ip + status = "active" + description = each.value.device + device_interface_id = netbox_device_interface.reality["${each.value.device}:${each.value.iface}"].id +} + +resource "netbox_inventory_item" "reality" { + for_each = local.reality_inventory + + device_id = netbox_device.this[each.value.device].id + name = each.value.name + description = each.value.description + part_id = each.value.part_id + serial = each.value.serial + discovered = true +} diff --git a/modules/infra/variables.tf b/modules/infra/variables.tf index b6099cf..2f8b587 100644 --- a/modules/infra/variables.tf +++ b/modules/infra/variables.tf @@ -66,6 +66,46 @@ variable "managed_ips" { default = [] } +variable "reality" { + description = <<-EOT + Per-host hardware reality keyed by device name, generated from pdbmux by + tools/backfill (see `make backfill`) — never hand-authored. It carries the + facts NetBox cannot learn from intent: serial/model/UUID, every recordable + interface (real NICs plus overlay/loopback/kube-lb) with MAC and CIDR IPs, + and CPU/RAM/disk inventory. Reality is wired into NetBox only for hosts that + also have an intent `devices/.yaml`; until that intent lands, a host's + reality sits reviewed-but-inert. + EOT + type = map(object({ + device = optional(string) + serial = optional(string) + model = optional(string) + uuid = optional(string) + interfaces = optional(list(object({ + name = string + mac = optional(string) + mtu = optional(number) + physical = optional(bool, false) + ips = optional(list(string), []) + })), []) + inventory = optional(object({ + cpu = optional(object({ + model = optional(string) + cores = optional(number) + count = optional(number) + })) + memory_bytes = optional(number) + disks = optional(list(object({ + name = string + model = optional(string) + serial = optional(string) + size_bytes = optional(number) + })), []) + })) + })) + default = {} +} + variable "vault_address" { description = "Vault server address for the token data source." type = string diff --git a/tools/backfill/emit.go b/tools/backfill/emit.go new file mode 100644 index 0000000..59e1eef --- /dev/null +++ b/tools/backfill/emit.go @@ -0,0 +1,108 @@ +package main + +import ( + "fmt" + "strings" +) + +// emitYAML renders a Reality as deterministic, yamllint-clean YAML consumed by +// modules/infra (variable "reality"). Field order is fixed and every list is +// pre-sorted upstream, so re-running the generator against unchanged facts +// produces byte-identical output (clean diffs). +func emitYAML(r Reality, source string) string { + var b strings.Builder + b.WriteString("# Generated by tools/backfill from pdbmux - do not edit by hand.\n") + b.WriteString("# " + source + "\n") + + line(&b, 0, "device", q(r.Device)) + if r.Serial != "" { + line(&b, 0, "serial", q(r.Serial)) + } + if r.Model != "" { + line(&b, 0, "model", q(r.Model)) + } + if r.UUID != "" { + line(&b, 0, "uuid", q(r.UUID)) + } + + if len(r.Interfaces) == 0 { + b.WriteString("interfaces: []\n") + } else { + b.WriteString("interfaces:\n") + for _, i := range r.Interfaces { + b.WriteString(" - name: " + q(i.Name) + "\n") + if i.MAC != "" { + line(&b, 4, "mac", q(i.MAC)) + } + if i.MTU > 0 { + line(&b, 4, "mtu", fmt.Sprintf("%d", i.MTU)) + } + line(&b, 4, "physical", fmt.Sprintf("%t", i.Physical)) + if len(i.IPs) == 0 { + line(&b, 4, "ips", "[]") + } else { + b.WriteString(" ips:\n") + for _, ip := range i.IPs { + b.WriteString(" - " + q(ip) + "\n") + } + } + } + } + + if r.Inventory != nil { + emitInventory(&b, r.Inventory) + } + return b.String() +} + +func emitInventory(b *strings.Builder, inv *Inventory) { + b.WriteString("inventory:\n") + if inv.CPU != nil { + b.WriteString(" cpu:\n") + if inv.CPU.Model != "" { + line(b, 4, "model", q(inv.CPU.Model)) + } + if inv.CPU.Cores > 0 { + line(b, 4, "cores", fmt.Sprintf("%d", inv.CPU.Cores)) + } + if inv.CPU.Count > 0 { + line(b, 4, "count", fmt.Sprintf("%d", inv.CPU.Count)) + } + } + if inv.MemoryBytes > 0 { + line(b, 2, "memory_bytes", fmt.Sprintf("%d", inv.MemoryBytes)) + } + if len(inv.Disks) == 0 { + line(b, 2, "disks", "[]") + } else { + b.WriteString(" disks:\n") + for _, d := range inv.Disks { + b.WriteString(" - name: " + q(d.Name) + "\n") + if d.Model != "" { + line(b, 6, "model", q(d.Model)) + } + if d.Serial != "" { + line(b, 6, "serial", q(d.Serial)) + } + if d.SizeBytes > 0 { + line(b, 6, "size_bytes", fmt.Sprintf("%d", d.SizeBytes)) + } + } + } +} + +func line(b *strings.Builder, indent int, key, val string) { + b.WriteString(strings.Repeat(" ", indent)) + b.WriteString(key) + b.WriteString(": ") + b.WriteString(val) + b.WriteByte('\n') +} + +// q double-quotes a scalar so values with ':' (MACs), '@'/'(' (CPU models) or +// '/' (serials, CIDRs) round-trip through yamldecode unambiguously. +func q(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `"`, `\"`) + return `"` + s + `"` +} diff --git a/tools/backfill/emit_test.go b/tools/backfill/emit_test.go new file mode 100644 index 0000000..68a8b72 --- /dev/null +++ b/tools/backfill/emit_test.go @@ -0,0 +1,79 @@ +package main + +import "testing" + +func sampleReality() Reality { + return Reality{ + Device: "prodnxsr0001", + Serial: "4RW6QM2", + Model: "OptiPlex 3050", + UUID: "4c4c4544-0052-5710-8036-b4c04f514d32", + Interfaces: []Interface{ + {Name: "enp2s0", MAC: "d8:9e:f3:75:c3:60", MTU: 1500, Physical: true, + IPs: []string{"198.18.15.1/24", "198.18.19.1/32"}}, + {Name: "loopback0", MAC: "f2:3d:ef:64:42:51", Physical: false, + IPs: []string{"198.18.19.1/32"}}, + }, + Inventory: &Inventory{ + CPU: &CPU{Model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz", Cores: 4, Count: 4}, + MemoryBytes: 33260900352, + Disks: []Disk{ + {Name: "sda", Model: "SSDSC2BX012T4N", Serial: "BTHC651303WB1P2OGN", SizeBytes: 1200243695616}, + }, + }, + } +} + +const wantYAML = `# Generated by tools/backfill from pdbmux - do not edit by hand. +# source: URL certname=prodnxsr0001.main.unkin.net +device: "prodnxsr0001" +serial: "4RW6QM2" +model: "OptiPlex 3050" +uuid: "4c4c4544-0052-5710-8036-b4c04f514d32" +interfaces: + - name: "enp2s0" + mac: "d8:9e:f3:75:c3:60" + mtu: 1500 + physical: true + ips: + - "198.18.15.1/24" + - "198.18.19.1/32" + - name: "loopback0" + mac: "f2:3d:ef:64:42:51" + physical: false + ips: + - "198.18.19.1/32" +inventory: + cpu: + model: "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + cores: 4 + count: 4 + memory_bytes: 33260900352 + disks: + - name: "sda" + model: "SSDSC2BX012T4N" + serial: "BTHC651303WB1P2OGN" + size_bytes: 1200243695616 +` + +func TestEmitYAMLGolden(t *testing.T) { + got := emitYAML(sampleReality(), "source: URL certname=prodnxsr0001.main.unkin.net") + if got != wantYAML { + t.Errorf("emit mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, wantYAML) + } +} + +func TestEmitYAMLDeterministic(t *testing.T) { + src := "source: URL certname=x" + if emitYAML(sampleReality(), src) != emitYAML(sampleReality(), src) { + t.Error("emit is not deterministic") + } +} + +func TestEmitYAMLEmptyCollections(t *testing.T) { + got := emitYAML(Reality{Device: "d"}, "s") + want := "# Generated by tools/backfill from pdbmux - do not edit by hand.\n# s\ndevice: \"d\"\ninterfaces: []\n" + if got != want { + t.Errorf("empty emit = %q, want %q", got, want) + } +} diff --git a/tools/backfill/go.mod b/tools/backfill/go.mod new file mode 100644 index 0000000..b7b1f9f --- /dev/null +++ b/tools/backfill/go.mod @@ -0,0 +1,3 @@ +module git.unkin.net/unkin/terraform-infra/tools/backfill + +go 1.25 diff --git a/tools/backfill/main.go b/tools/backfill/main.go new file mode 100644 index 0000000..c060472 --- /dev/null +++ b/tools/backfill/main.go @@ -0,0 +1,81 @@ +// Command backfill queries pdbmux (the PuppetDB multiplexer) for already- +// provisioned hosts and emits their hardware reality — serial, model, UUID, +// every recordable interface's MAC/IPs (including overlay/loopback/kube-lb +// addresses) and CPU/RAM/disk inventory — as reviewable per-host YAML for the +// NetBox reality side (modules/infra variable "reality"). +// +// It is a generator, not a live data source: it writes committed config that +// terraform later reconciles into NetBox. Intent (config/.../devices/*.yaml) +// stays human-authored; this fills reality only. +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + defaultURL = "https://pdbmux.k8s.syd1.au.unkin.net/pdb/query/v4/facts" + defaultDomain = "main.unkin.net" +) + +func main() { + url := flag.String("url", envOr("PDBMUX_URL", defaultURL), "pdbmux PuppetDB v4 facts endpoint") + out := flag.String("out", "", "directory to write .yaml into (default: stdout)") + domain := flag.String("domain", defaultDomain, "domain appended to short hostnames to form certnames") + flag.Parse() + + hosts := flag.Args() + if len(hosts) == 0 { + fmt.Fprintln(os.Stderr, "usage: backfill [--url URL] [--out DIR] [--domain D] host...") + os.Exit(2) + } + + client := newFactsClient(*url) + rc := 0 + for _, h := range hosts { + certname := h + if !strings.Contains(h, ".") { + certname = h + "." + *domain + } + facts, err := client.facts(certname) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: query failed: %v\n", certname, err) + rc = 1 + continue + } + if len(facts) == 0 { + fmt.Fprintf(os.Stderr, "%s: no facts (not in pdbmux)\n", certname) + continue + } + + text := emitYAML(factsToReality(certname, facts), fmt.Sprintf("source: %s certname=%s", *url, certname)) + if *out == "" { + fmt.Print(text) + continue + } + if err := os.MkdirAll(*out, 0o755); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", certname, err) + rc = 1 + continue + } + path := filepath.Join(*out, shortName(certname)+".yaml") + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", certname, err) + rc = 1 + continue + } + fmt.Fprintf(os.Stderr, "wrote %s\n", path) + } + os.Exit(rc) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/tools/backfill/pdbmux.go b/tools/backfill/pdbmux.go new file mode 100644 index 0000000..c246563 --- /dev/null +++ b/tools/backfill/pdbmux.go @@ -0,0 +1,61 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// factRecord is one PuppetDB v4 /facts row as served by pdbmux. +type factRecord struct { + Certname string `json:"certname"` + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + +// factsClient queries a pdbmux (PuppetDB v4) facts endpoint. pdbmux merges the +// old (Consul) and new (k8s) PuppetDBs and needs no auth, so a plain GET with a +// certname AST query is all that is required. +type factsClient struct { + url string + client *http.Client +} + +func newFactsClient(factsURL string) *factsClient { + return &factsClient{url: factsURL, client: &http.Client{Timeout: 30 * time.Second}} +} + +// facts returns the certname's facts as a name->value map. A node pdbmux does +// not know about yields an empty map and a nil error. +func (c *factsClient) facts(certname string) (map[string]json.RawMessage, error) { + q, _ := json.Marshal([]any{"=", "certname", certname}) + params := url.Values{} + params.Set("query", string(q)) + + resp, err := c.client.Get(c.url + "?" + params.Encode()) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + var rows []factRecord + if err := json.Unmarshal(body, &rows); err != nil { + return nil, fmt.Errorf("decode facts: %w", err) + } + out := make(map[string]json.RawMessage, len(rows)) + for _, r := range rows { + out[r.Name] = r.Value + } + return out, nil +} diff --git a/tools/backfill/pdbmux_test.go b/tools/backfill/pdbmux_test.go new file mode 100644 index 0000000..4c2f283 --- /dev/null +++ b/tools/backfill/pdbmux_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestFactsClientParsesResponse(t *testing.T) { + var gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query().Get("query") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[ + {"certname":"h.main.unkin.net","name":"dmi","value":{"product":{"name":"OptiPlex 3060","serial_number":"S1","uuid":"U1"}},"environment":"production"}, + {"certname":"h.main.unkin.net","name":"uuid","value":"U1","environment":"production"} + ]`)) + })) + defer srv.Close() + + facts, err := newFactsClient(srv.URL).facts("h.main.unkin.net") + if err != nil { + t.Fatalf("facts: %v", err) + } + if gotQuery != `["=","certname","h.main.unkin.net"]` { + t.Errorf("query = %s", gotQuery) + } + if _, ok := facts["dmi"]; !ok { + t.Errorf("dmi fact missing: %v", facts) + } + + r := factsToReality("h.main.unkin.net", facts) + if r.Model != "OptiPlex 3060" || r.Serial != "S1" { + t.Errorf("reality = %+v", r) + } +} + +func TestFactsClientEmptyForUnknownNode(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + + facts, err := newFactsClient(srv.URL).facts("nope") + if err != nil { + t.Fatalf("facts: %v", err) + } + if len(facts) != 0 { + t.Errorf("expected empty facts, got %v", facts) + } +} + +func TestFactsClientErrorsOnNon200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "all backends failed", http.StatusBadGateway) + })) + defer srv.Close() + + if _, err := newFactsClient(srv.URL).facts("h"); err == nil { + t.Fatal("expected error on 502") + } +} diff --git a/tools/backfill/reality.go b/tools/backfill/reality.go new file mode 100644 index 0000000..2260125 --- /dev/null +++ b/tools/backfill/reality.go @@ -0,0 +1,271 @@ +package main + +import ( + "encoding/json" + "fmt" + "net" + "sort" + "strings" +) + +// placeholderMAC is the constant MAC the kernel assigns to Calico veths; it is +// never a real hardware address and must not be recorded. +const placeholderMAC = "ee:ee:ee:ee:ee:ee" + +// virtualDiskPrefixes are block-device name prefixes that are not physical +// disks (Ceph RBD volumes, loop/device-mapper/optical/ram devices). They come +// and go with workloads, so recording them would make the backfill non-idempotent. +var virtualDiskPrefixes = []string{"rbd", "loop", "dm-", "md", "sr", "zram", "ram", "fd", "nbd", "zd", "dasd"} + +// Interface is one NIC's recorded reality. +type Interface struct { + Name string + MAC string + MTU int + Physical bool + IPs []string // CIDR form, e.g. 198.18.15.1/24 +} + +// CPU, Disk and Inventory carry the hardware inventory NetBox never learns from +// intent. +type CPU struct { + Model string + Cores int + Count int +} + +type Disk struct { + Name string + Model string + Serial string + SizeBytes int64 +} + +type Inventory struct { + CPU *CPU + MemoryBytes int64 + Disks []Disk +} + +// Reality is the hardware-owned truth a running host reports, destined for the +// NetBox reality side (interfaces + inventory). +type Reality struct { + Device string + Serial string + Model string + UUID string + Interfaces []Interface + Inventory *Inventory +} + +// factBinding is one address binding under networking.interfaces..bindings +// (IPv4) or .bindings6 (IPv6). +type factBinding struct { + Address string `json:"address"` + Netmask string `json:"netmask"` +} + +// shortName returns the leading label of a certname (prodnxsr0001.main… -> +// prodnxsr0001), which is the NetBox device name. +func shortName(certname string) string { + if i := strings.IndexByte(certname, '.'); i >= 0 { + return certname[:i] + } + return certname +} + +// factsToReality maps a Facter/PuppetDB factset to the reality we record. +func factsToReality(certname string, facts map[string]json.RawMessage) Reality { + r := Reality{Device: shortName(certname)} + + var dmi struct { + Product struct { + Name string `json:"name"` + UUID string `json:"uuid"` + Serial string `json:"serial_number"` + } `json:"product"` + } + getFact(facts, "dmi", &dmi) + r.Serial = firstNonEmpty(dmi.Product.Serial, getStringFact(facts, "serialnumber")) + r.Model = firstNonEmpty(dmi.Product.Name, getStringFact(facts, "productname")) + r.UUID = firstNonEmpty(dmi.Product.UUID, getStringFact(facts, "uuid")) + + r.Interfaces = interfaces(facts) + r.Inventory = inventory(facts) + return r +} + +// interfaces enumerates recordable NICs. A physical NIC is always kept (it is +// real hardware even when down); a virtual interface is kept only when it holds +// a routable address, which naturally drops the ephemeral Calico veths (link- +// local only) and lo (loopback only) while keeping overlay/loopback/kube-lb +// interfaces. Interface names are never assumed — they come straight from Facter. +func interfaces(facts map[string]json.RawMessage) []Interface { + var networking struct { + Interfaces map[string]struct { + MAC string `json:"mac"` + MTU int `json:"mtu"` + Physical bool `json:"physical"` + Bindings []factBinding `json:"bindings"` + Bindings6 []factBinding `json:"bindings6"` + } `json:"interfaces"` + } + getFact(facts, "networking", &networking) + + var out []Interface + for name, iface := range networking.Interfaces { + var ips []string + for _, b := range append(append([]factBinding{}, iface.Bindings...), iface.Bindings6...) { + ip := net.ParseIP(b.Address) + if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsUnspecified() { + continue + } + ips = append(ips, toCIDR(b.Address, b.Netmask)) + } + if !iface.Physical && len(ips) == 0 { + continue + } + mac := iface.MAC + if strings.EqualFold(mac, placeholderMAC) { + mac = "" + } + sort.Strings(ips) + out = append(out, Interface{ + Name: name, + MAC: mac, + MTU: iface.MTU, + Physical: iface.Physical, + IPs: dedupe(ips), + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// inventory collects CPU/RAM/disk reality. Disks with a virtual-device name are +// skipped so the output only reflects physical hardware. +func inventory(facts map[string]json.RawMessage) *Inventory { + var processors struct { + Count int `json:"count"` + Cores int `json:"cores"` + Models []string `json:"models"` + } + var memory struct { + System struct { + TotalBytes int64 `json:"total_bytes"` + } `json:"system"` + } + var disks map[string]struct { + Model string `json:"model"` + Serial string `json:"serial"` + SizeBytes int64 `json:"size_bytes"` + } + getFact(facts, "processors", &processors) + getFact(facts, "memory", &memory) + getFact(facts, "disks", &disks) + + inv := &Inventory{MemoryBytes: memory.System.TotalBytes} + if len(processors.Models) > 0 || processors.Count > 0 { + model := "" + if len(processors.Models) > 0 { + model = processors.Models[0] + } + inv.CPU = &CPU{Model: model, Cores: processors.Cores, Count: processors.Count} + } + + names := make([]string, 0, len(disks)) + for n := range disks { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + if isVirtualDisk(n) { + continue + } + d := disks[n] + inv.Disks = append(inv.Disks, Disk{Name: n, Model: d.Model, Serial: d.Serial, SizeBytes: d.SizeBytes}) + } + + if inv.CPU == nil && inv.MemoryBytes == 0 && len(inv.Disks) == 0 { + return nil + } + return inv +} + +func isVirtualDisk(name string) bool { + for _, p := range virtualDiskPrefixes { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} + +// toCIDR renders address + netmask as address/prefixlen. Facter secondary +// bindings can omit the netmask; those default to a host route (/32 or /128). +func toCIDR(addr, netmask string) string { + return fmt.Sprintf("%s/%d", addr, maskToPrefix(addr, netmask)) +} + +func maskToPrefix(addr, netmask string) int { + v4 := net.ParseIP(addr).To4() != nil + hostBits := func() int { + if v4 { + return 32 + } + return 128 + } + if netmask == "" { + return hostBits() + } + m := net.ParseIP(netmask) + if m == nil { + return hostBits() + } + var mask net.IPMask + if v4 { + mask = net.IPMask(m.To4()) + } else { + mask = net.IPMask(m.To16()) + } + ones, bits := mask.Size() + if bits == 0 { // non-canonical mask + return hostBits() + } + return ones +} + +func getFact(facts map[string]json.RawMessage, name string, dst any) { + if raw, ok := facts[name]; ok { + _ = json.Unmarshal(raw, dst) + } +} + +func getStringFact(facts map[string]json.RawMessage, name string) string { + var s string + getFact(facts, name, &s) + return s +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +func dedupe(in []string) []string { + if len(in) == 0 { + return in + } + out := in[:1] + for _, v := range in[1:] { + if v != out[len(out)-1] { + out = append(out, v) + } + } + return out +} diff --git a/tools/backfill/reality_test.go b/tools/backfill/reality_test.go new file mode 100644 index 0000000..a2fdb57 --- /dev/null +++ b/tools/backfill/reality_test.go @@ -0,0 +1,168 @@ +package main + +import ( + "encoding/json" + "os" + "testing" +) + +// loadFixture reads a PuppetDB /facts response fixture into a name->value map. +func loadFixture(t *testing.T, path string) map[string]json.RawMessage { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var rows []factRecord + if err := json.Unmarshal(data, &rows); err != nil { + t.Fatalf("decode fixture: %v", err) + } + m := make(map[string]json.RawMessage, len(rows)) + for _, r := range rows { + m[r.Name] = r.Value + } + return m +} + +func realityFor(t *testing.T) Reality { + t.Helper() + facts := loadFixture(t, "testdata/facts_prodnxsr0001.json") + return factsToReality("prodnxsr0001.main.unkin.net", facts) +} + +func TestFactsToRealityIdentity(t *testing.T) { + r := realityFor(t) + if r.Device != "prodnxsr0001" { + t.Errorf("device = %q, want prodnxsr0001", r.Device) + } + if r.Serial != "4RW6QM2" { + t.Errorf("serial = %q, want 4RW6QM2", r.Serial) + } + if r.Model != "OptiPlex 3050" { + t.Errorf("model = %q, want OptiPlex 3050", r.Model) + } + if r.UUID != "4c4c4544-0052-5710-8036-b4c04f514d32" { + t.Errorf("uuid = %q", r.UUID) + } +} + +func TestInterfaceFiltering(t *testing.T) { + r := realityFor(t) + got := map[string]Interface{} + for _, i := range r.Interfaces { + got[i.Name] = i + } + + // Real NICs, overlay, loopback and kube-lb interfaces are kept. + for _, want := range []string{"enp2s0", "enp3s0", "loopback0", "loopback1", "loopback2", "kube-lb0", "flannel.1"} { + if _, ok := got[want]; !ok { + t.Errorf("expected interface %q to be kept", want) + } + } + // lo (loopback only) and Calico veths (link-local only) are dropped. + for _, drop := range []string{"lo", "calid8d62485b84", "cali636724c6fe6"} { + if _, ok := got[drop]; ok { + t.Errorf("interface %q should have been dropped", drop) + } + } + // Placeholder Calico MAC never leaks onto a kept interface. + for _, i := range r.Interfaces { + if i.MAC == placeholderMAC { + t.Errorf("interface %q kept placeholder MAC", i.Name) + } + } +} + +func TestInterfaceAddresses(t *testing.T) { + r := realityFor(t) + got := map[string]Interface{} + for _, i := range r.Interfaces { + got[i.Name] = i + } + + // enp2s0: primary /24 plus an unmasked secondary that defaults to /32. + assertIPs(t, "enp2s0", got["enp2s0"].IPs, []string{"198.18.15.1/24", "198.18.19.1/32"}) + // kube-lb0: eight VIPs, all /32, sorted and deduped. + assertIPs(t, "kube-lb0", got["kube-lb0"].IPs, []string{ + "198.18.199.0/32", "198.18.200.0/32", "198.18.200.1/32", "198.18.200.2/32", + "198.18.200.3/32", "198.18.200.4/32", "198.18.200.5/32", "198.18.200.9/32", + }) + // flannel.1: overlay /32. + assertIPs(t, "flannel.1", got["flannel.1"].IPs, []string{"10.42.0.0/32"}) + + if !got["enp2s0"].Physical { + t.Error("enp2s0 should be physical") + } + if got["loopback0"].Physical { + t.Error("loopback0 should be virtual") + } +} + +func assertIPs(t *testing.T, iface string, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s ips = %v, want %v", iface, got, want) + return + } + for i := range want { + if got[i] != want[i] { + t.Errorf("%s ips = %v, want %v", iface, got, want) + return + } + } +} + +func TestInventory(t *testing.T) { + r := realityFor(t) + if r.Inventory == nil { + t.Fatal("inventory is nil") + } + if r.Inventory.MemoryBytes != 33260900352 { + t.Errorf("memory = %d", r.Inventory.MemoryBytes) + } + if r.Inventory.CPU == nil || r.Inventory.CPU.Model != "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" { + t.Errorf("cpu = %+v", r.Inventory.CPU) + } + if r.Inventory.CPU.Count != 4 { + t.Errorf("cpu count = %d, want 4", r.Inventory.CPU.Count) + } + + names := map[string]Disk{} + for _, d := range r.Inventory.Disks { + names[d.Name] = d + } + // Physical disks kept; Ceph RBD volumes dropped. + if _, ok := names["sda"]; !ok { + t.Error("sda should be kept") + } + if _, ok := names["nvme0n1"]; !ok { + t.Error("nvme0n1 should be kept") + } + for _, drop := range []string{"rbd0", "rbd1", "rbd5"} { + if _, ok := names[drop]; ok { + t.Errorf("virtual disk %q should be dropped", drop) + } + } + if names["sda"].SizeBytes != 1200243695616 || names["sda"].Serial != "BTHC651303WB1P2OGN" { + t.Errorf("sda = %+v", names["sda"]) + } +} + +func TestMaskToPrefix(t *testing.T) { + cases := []struct { + addr, mask string + want int + }{ + {"198.18.15.1", "255.255.255.0", 24}, + {"198.18.19.1", "255.255.255.255", 32}, + {"198.18.19.1", "", 32}, // unmasked secondary + {"198.18.19.1", "garbage", 32}, // unparseable mask + {"fd00::1", "", 128}, // v6 default + {"fd00::1", "ffff:ffff:ffff:ffff::", 64}, + } + for _, c := range cases { + if got := maskToPrefix(c.addr, c.mask); got != c.want { + t.Errorf("maskToPrefix(%q,%q) = %d, want %d", c.addr, c.mask, got, c.want) + } + } +} diff --git a/tools/backfill/testdata/facts_prodnxsr0001.json b/tools/backfill/testdata/facts_prodnxsr0001.json new file mode 100644 index 0000000..0842680 --- /dev/null +++ b/tools/backfill/testdata/facts_prodnxsr0001.json @@ -0,0 +1,461 @@ +[ + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "dmi", + "value": { + "bios": { + "release_date": "09/19/2023", + "vendor": "Dell Inc.", + "version": "1.27.0" + }, + "board": { + "manufacturer": "Dell Inc.", + "product": "0JP3NX", + "serial_number": "/4RW6QM2/CNPE1007AP0J4Q/" + }, + "chassis": { + "type": "Desktop" + }, + "manufacturer": "Dell Inc.", + "product": { + "name": "OptiPlex 3050", + "serial_number": "4RW6QM2", + "uuid": "4c4c4544-0052-5710-8036-b4c04f514d32" + } + } + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "networking", + "value": { + "domain": "main.unkin.net", + "fqdn": "prodnxsr0001.main.unkin.net", + "hostname": "prodnxsr0001", + "interfaces": { + "cali636724c6fe6": { + "bindings6": [ + { + "address": "fe80::ecee:eeff:feee:eeee", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "ip6": "fe80::ecee:eeff:feee:eeee", + "mac": "ee:ee:ee:ee:ee:ee", + "mtu": 1450, + "netmask6": "ffff:ffff:ffff:ffff::", + "network6": "fe80::", + "operational_state": "up", + "physical": false, + "scope6": "link" + }, + "calid8d62485b84": { + "bindings6": [ + { + "address": "fe80::ecee:eeff:feee:eeee", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "ip6": "fe80::ecee:eeff:feee:eeee", + "mac": "ee:ee:ee:ee:ee:ee", + "mtu": 1450, + "netmask6": "ffff:ffff:ffff:ffff::", + "network6": "fe80::", + "operational_state": "up", + "physical": false, + "scope6": "link" + }, + "enp2s0": { + "bindings": [ + { + "address": "198.18.15.1", + "netmask": "255.255.255.0", + "network": "198.18.15.0" + }, + { + "address": "198.18.19.1" + } + ], + "bindings6": [ + { + "address": "fe80::da9e:f3ff:fe75:c360", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "duplex": "full", + "ip": "198.18.15.1", + "ip6": "fe80::da9e:f3ff:fe75:c360", + "mac": "d8:9e:f3:75:c3:60", + "mtu": 1500, + "netmask": "255.255.255.0", + "netmask6": "ffff:ffff:ffff:ffff::", + "network": "198.18.15.0", + "network6": "fe80::", + "operational_state": "up", + "physical": true, + "scope6": "link", + "speed": 1000 + }, + "enp3s0": { + "bindings": [ + { + "address": "198.18.21.1", + "netmask": "255.255.255.0", + "network": "198.18.21.0" + }, + { + "address": "198.18.19.1" + } + ], + "bindings6": [ + { + "address": "fe80::2ac:d0ff:fe00:50", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "duplex": "full", + "ip": "198.18.21.1", + "ip6": "fe80::2ac:d0ff:fe00:50", + "mac": "00:ac:d0:00:00:50", + "mtu": 1500, + "netmask": "255.255.255.0", + "netmask6": "ffff:ffff:ffff:ffff::", + "network": "198.18.21.0", + "network6": "fe80::", + "operational_state": "up", + "physical": true, + "scope6": "link", + "speed": 2500 + }, + "flannel.1": { + "bindings": [ + { + "address": "10.42.0.0", + "netmask": "255.255.255.255", + "network": "10.42.0.0" + } + ], + "bindings6": [ + { + "address": "fe80::54ac:21ff:fe3e:cd9b", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "ip": "10.42.0.0", + "ip6": "fe80::54ac:21ff:fe3e:cd9b", + "mac": "56:ac:21:3e:cd:9b", + "mtu": 1450, + "netmask": "255.255.255.255", + "netmask6": "ffff:ffff:ffff:ffff::", + "network": "10.42.0.0", + "network6": "fe80::", + "operational_state": "unknown", + "physical": false, + "scope6": "link" + }, + "kube-lb0": { + "bindings": [ + { + "address": "198.18.200.2", + "netmask": "255.255.255.255", + "network": "198.18.200.2" + }, + { + "address": "198.18.200.3", + "netmask": "255.255.255.255", + "network": "198.18.200.3" + }, + { + "address": "198.18.200.1", + "netmask": "255.255.255.255", + "network": "198.18.200.1" + }, + { + "address": "198.18.199.0", + "netmask": "255.255.255.255", + "network": "198.18.199.0" + }, + { + "address": "198.18.200.4", + "netmask": "255.255.255.255", + "network": "198.18.200.4" + }, + { + "address": "198.18.200.5", + "netmask": "255.255.255.255", + "network": "198.18.200.5" + }, + { + "address": "198.18.200.0", + "netmask": "255.255.255.255", + "network": "198.18.200.0" + }, + { + "address": "198.18.200.9", + "netmask": "255.255.255.255", + "network": "198.18.200.9" + } + ], + "bindings6": [ + { + "address": "fe80::1c4f:62ff:fe0c:e8e1", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff::", + "network": "fe80::", + "scope6": "link" + } + ], + "ip": "198.18.200.2", + "ip6": "fe80::1c4f:62ff:fe0c:e8e1", + "mac": "1e:4f:62:0c:e8:e1", + "mtu": 1500, + "netmask": "255.255.255.255", + "netmask6": "ffff:ffff:ffff:ffff::", + "network": "198.18.200.2", + "network6": "fe80::", + "operational_state": "unknown", + "physical": false, + "scope6": "link" + }, + "lo": { + "bindings": [ + { + "address": "127.0.0.1", + "netmask": "255.0.0.0", + "network": "127.0.0.0" + } + ], + "bindings6": [ + { + "address": "::1", + "flags": [ + "permanent" + ], + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "network": "::1", + "scope6": "host" + } + ], + "ip": "127.0.0.1", + "ip6": "::1", + "mtu": 65536, + "netmask": "255.0.0.0", + "netmask6": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "network": "127.0.0.0", + "network6": "::1", + "operational_state": "unknown", + "physical": false, + "scope6": "host" + }, + "loopback0": { + "bindings": [ + { + "address": "198.18.19.1", + "netmask": "255.255.255.255", + "network": "198.18.19.1" + } + ], + "ip": "198.18.19.1", + "mac": "f2:3d:ef:64:42:51", + "mtu": 1500, + "netmask": "255.255.255.255", + "network": "198.18.19.1", + "operational_state": "unknown", + "physical": false + }, + "loopback1": { + "bindings": [ + { + "address": "198.18.22.1", + "netmask": "255.255.255.255", + "network": "198.18.22.1" + } + ], + "ip": "198.18.22.1", + "mac": "aa:36:f9:d5:f0:1d", + "mtu": 1500, + "netmask": "255.255.255.255", + "network": "198.18.22.1", + "operational_state": "unknown", + "physical": false + }, + "loopback2": { + "bindings": [ + { + "address": "198.18.23.1", + "netmask": "255.255.255.255", + "network": "198.18.23.1" + } + ], + "ip": "198.18.23.1", + "mac": "96:aa:ea:a3:6d:c5", + "mtu": 1500, + "netmask": "255.255.255.255", + "network": "198.18.23.1", + "operational_state": "unknown", + "physical": false + } + }, + "ip": "198.18.15.1", + "ip6": "fe80::da9e:f3ff:fe75:c360", + "mac": "d8:9e:f3:75:c3:60", + "mtu": 1500, + "netmask": "255.255.255.0", + "netmask6": "ffff:ffff:ffff:ffff::", + "network": "198.18.15.0", + "network6": "fe80::", + "primary": "enp2s0", + "scope6": "link" + } + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "processors", + "value": { + "cores": 4, + "count": 4, + "extensions": [ + "x86_64", + "x86_64-v1", + "x86_64-v2", + "x86_64-v3" + ], + "isa": "x86_64", + "models": [ + "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz", + "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz", + "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz", + "Intel(R) Core(TM) i5-7500T CPU @ 2.70GHz" + ], + "physicalcount": 1, + "speed": "3.10 GHz", + "threads": 1 + } + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "memory", + "value": { + "swap": { + "available": "1.97 GiB", + "available_bytes": 2115760128, + "capacity": "1.48%", + "total": "2.00 GiB", + "total_bytes": 2147479552, + "used": "30.25 MiB", + "used_bytes": 31719424 + }, + "system": { + "available": "15.74 GiB", + "available_bytes": 16900333568, + "capacity": "49.19%", + "total": "30.98 GiB", + "total_bytes": 33260900352, + "used": "15.24 GiB", + "used_bytes": 16360566784 + } + } + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "disks", + "value": { + "nvme0n1": { + "model": "IM2P33F8-256GD1", + "serial": "2M3029AD7KFE", + "size": "238.47 GiB", + "size_bytes": 256060514304, + "type": "ssd", + "wwn": "nvme.1cc1-324d333032394144374b4645-494d3250333346382d323536474431-00000001" + }, + "rbd0": { + "size": "10.00 GiB", + "size_bytes": 10737418240, + "type": "ssd" + }, + "rbd1": { + "size": "1.00 GiB", + "size_bytes": 1073741824, + "type": "ssd" + }, + "rbd2": { + "size": "10.00 GiB", + "size_bytes": 10737418240, + "type": "ssd" + }, + "rbd3": { + "size": "10.00 GiB", + "size_bytes": 10737418240, + "type": "ssd" + }, + "rbd4": { + "size": "10.00 GiB", + "size_bytes": 10737418240, + "type": "ssd" + }, + "rbd5": { + "size": "10.00 GiB", + "size_bytes": 10737418240, + "type": "ssd" + }, + "sda": { + "model": "SSDSC2BX012T4N", + "serial": "BTHC651303WB1P2OGN", + "size": "1.09 TiB", + "size_bytes": 1200243695616, + "type": "ssd", + "vendor": "ATA", + "wwn": "0x55cd2e414d94c8ff" + } + } + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "serialnumber", + "value": "4RW6QM2" + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "productname", + "value": "OptiPlex 3050" + }, + { + "certname": "prodnxsr0001.main.unkin.net", + "environment": "develop", + "name": "uuid", + "value": "4c4c4544-0052-5710-8036-b4c04f514d32" + } +]