b0e3e31a6b
## Why After v0.1.4 pointed HA peer URLs at per-pod ClusterIP Services, kea-dhcp4 2.6 starts the HA service but then crashes at hook load: DHCP4_CONFIG_LOAD_FAIL ... Error initializing hooks: CmdHttpListener::run failed: unable to setup TCP acceptor for listening to the incoming HTTP requests: bind: Cannot assign requested address With core multi-threading enabled (Kea 2.6 default), the HA hook opens a dedicated HTTP listener bound to *this* server's peer url address. That address is now a per-pod ClusterIP — virtual (kube-proxy DNAT), not assignable on the pod — so the bind fails. Peers must be reached via ClusterIP, but the local listener must bind a pod-local address. ## How - Set the HA relationship's multi-threading block http-dedicated-listener:false (enable-multi-threading:true). Per Kea 2.6 docs this makes inbound HA traffic flow through kea-ctrl-agent instead of a hook-owned listener. The ctrl-agent sidecar already binds 0.0.0.0:8000, and the per-pod Service targetPort 8000 routes ClusterIP:8000 to that container, so remote peers keep reaching this server at its ClusterIP while nothing binds the virtual address locally. - Regression tests: rendered config disables the dedicated listener (string + parsed high-availability.multi-threading assertion); ctrl-agent binds 0.0.0.0 (the pod-local address the CA-mediated route depends on).
343 lines
11 KiB
Go
343 lines
11 KiB
Go
package kea
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
// referenceInput mirrors the ISC dhcpd config that must be fully representable:
|
|
// subnets 198.18.13-17.0/24 (pool .200-.220, routers, dns .19.15, next-server
|
|
// .19.19, domain main.unkin.net), plus 198.18.25.0/24 with no pool; the two PXE
|
|
// arch classes; authoritative; ddns off.
|
|
func referenceInput() RenderInput {
|
|
cluster := v1alpha1.KeaCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe"},
|
|
Spec: v1alpha1.KeaClusterSpec{
|
|
DomainName: "main.unkin.net",
|
|
DefaultLeaseTime: 1200,
|
|
MaxLeaseTime: 86400,
|
|
HA: v1alpha1.HASpec{Mode: v1alpha1.HAHotStandby},
|
|
},
|
|
}
|
|
|
|
mkSubnet := func(name, cidr string, withPool bool) v1alpha1.KeaSubnet {
|
|
s := v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: name},
|
|
Spec: v1alpha1.KeaSubnetSpec{
|
|
Subnet: cidr,
|
|
Routers: []string{strings.TrimSuffix(cidr, "0/24") + "1"},
|
|
DNSServers: []string{"198.18.19.15"},
|
|
DomainName: "main.unkin.net",
|
|
NextServer: "198.18.19.19",
|
|
},
|
|
}
|
|
if withPool {
|
|
base := strings.TrimSuffix(cidr, "0/24")
|
|
s.Spec.Pools = []string{base + "200 - " + base + "220"}
|
|
}
|
|
return s
|
|
}
|
|
|
|
subnets := []v1alpha1.KeaSubnet{
|
|
mkSubnet("s13", "198.18.13.0/24", true),
|
|
mkSubnet("s14", "198.18.14.0/24", true),
|
|
mkSubnet("s15", "198.18.15.0/24", true),
|
|
mkSubnet("s16", "198.18.16.0/24", true),
|
|
mkSubnet("s17", "198.18.17.0/24", true),
|
|
mkSubnet("s25", "198.18.25.0/24", false),
|
|
}
|
|
|
|
classes := []v1alpha1.KeaClientClass{
|
|
{ObjectMeta: metav1.ObjectMeta{Name: "Legacy"}, Spec: v1alpha1.KeaClientClassSpec{
|
|
ArchHex: []string{"0x0000"}, BootFileName: "/undionly.kpxe"}},
|
|
{ObjectMeta: metav1.ObjectMeta{Name: "UEFI-64"}, Spec: v1alpha1.KeaClientClassSpec{
|
|
ArchHex: []string{"0x0007", "0x0009"}, BootFileName: "/ipxe.efi"}},
|
|
}
|
|
|
|
peers := []Peer{
|
|
{Name: "server0", URL: "http://pxe-0.pxe-headless.dhcp-system:8000/", Role: "primary"},
|
|
{Name: "server1", URL: "http://pxe-1.pxe-headless.dhcp-system:8000/", Role: "standby"},
|
|
}
|
|
|
|
return RenderInput{Cluster: cluster, Subnets: subnets, ClientClasses: classes, Peers: peers}
|
|
}
|
|
|
|
func TestRenderDHCP4IsValidJSON(t *testing.T) {
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
var root map[string]any
|
|
if err := json.Unmarshal([]byte(out), &root); err != nil {
|
|
t.Fatalf("output is not valid JSON: %v\n%s", err, out)
|
|
}
|
|
if _, ok := root["Dhcp4"]; !ok {
|
|
t.Fatalf("missing Dhcp4 top-level key")
|
|
}
|
|
}
|
|
|
|
func TestRenderDHCP4ReferenceSemantics(t *testing.T) {
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
must := []string{
|
|
`"authoritative": true`,
|
|
`"ddns-send-updates": false`,
|
|
`"valid-lifetime": 1200`,
|
|
`"max-valid-lifetime": 86400`,
|
|
`"198.18.13.0/24"`,
|
|
`"198.18.25.0/24"`,
|
|
`"198.18.13.200 - 198.18.13.220"`,
|
|
`"next-server": "198.18.19.19"`,
|
|
`"data": "198.18.19.15"`, // domain-name-servers
|
|
`"data": "198.18.13.1"`, // routers
|
|
`"data": "main.unkin.net"`, // domain-name
|
|
`"boot-file-name": "/undionly.kpxe"`,
|
|
`"boot-file-name": "/ipxe.efi"`,
|
|
`option[93].hex == 0x0000`,
|
|
`option[93].hex == 0x0007 or option[93].hex == 0x0009`,
|
|
`libdhcp_ha.so`,
|
|
`libdhcp_lease_cmds.so`,
|
|
`"mode": "hot-standby"`,
|
|
ThisServerPlaceholder,
|
|
`memfile`,
|
|
}
|
|
for _, m := range must {
|
|
if !strings.Contains(out, m) {
|
|
t.Errorf("rendered config missing %q\n---\n%s", m, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestHADedicatedListenerDisabled guards the HA listener/bind fix: with Kea 2.6
|
|
// core multi-threading on by default, the HA hook would otherwise open a
|
|
// dedicated HTTP listener bound to this server's peer url — a per-pod ClusterIP
|
|
// that is virtual (kube-proxy DNAT) and unassignable on the pod, failing with
|
|
// "Cannot assign requested address". The rendered config must disable the
|
|
// dedicated listener so inbound HA traffic is served by kea-ctrl-agent
|
|
// (0.0.0.0:8000) while peers stay reachable at their ClusterIP:8000.
|
|
func TestHADedicatedListenerDisabled(t *testing.T) {
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
if !strings.Contains(out, `"http-dedicated-listener": false`) {
|
|
t.Errorf("HA hook must disable the dedicated HTTP listener (it binds this server's ClusterIP peer url, unassignable on the pod); got:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, `"enable-multi-threading": true`) {
|
|
t.Errorf("HA multi-threading must stay enabled (CA-mediated HA traffic); got:\n%s", out)
|
|
}
|
|
|
|
// The CA-mediated route only works because the dedicated listener stays off
|
|
// on the same port the ctrl-agent binds: parse out the HA hook and assert it.
|
|
var root dhcp4Root
|
|
if err := json.Unmarshal([]byte(out), &root); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
var haParams map[string]any
|
|
for _, h := range root.Dhcp4.HooksLibraries {
|
|
if strings.Contains(h.Library, "libdhcp_ha.so") {
|
|
haParams = h.Parameters
|
|
}
|
|
}
|
|
if haParams == nil {
|
|
t.Fatal("HA hook library not present in rendered config")
|
|
}
|
|
rels, ok := haParams["high-availability"].([]any)
|
|
if !ok || len(rels) != 1 {
|
|
t.Fatalf("high-availability block malformed: %#v", haParams["high-availability"])
|
|
}
|
|
mt, ok := rels[0].(map[string]any)["multi-threading"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("HA relationship missing multi-threading block: %#v", rels[0])
|
|
}
|
|
if mt["http-dedicated-listener"] != false {
|
|
t.Errorf("http-dedicated-listener must be false, got %#v", mt["http-dedicated-listener"])
|
|
}
|
|
}
|
|
|
|
// TestSubnetWithoutPoolIsDeclared verifies the pool-less subnet still appears
|
|
// (Kea must know the subnet to service relayed requests) but carries no pools.
|
|
func TestSubnetWithoutPoolIsDeclared(t *testing.T) {
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
var root dhcp4Root
|
|
if err := json.Unmarshal([]byte(out), &root); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
var found bool
|
|
for _, s := range root.Dhcp4.Subnet4 {
|
|
if s.Subnet == "198.18.25.0/24" {
|
|
found = true
|
|
if len(s.Pools) != 0 {
|
|
t.Errorf("198.18.25.0/24 should have no pools, got %v", s.Pools)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("pool-less subnet 198.18.25.0/24 not declared")
|
|
}
|
|
}
|
|
|
|
// TestRenderDeterministicWithShuffledInput asserts byte-identical output
|
|
// regardless of input ordering — unsorted input would churn the ConfigMap and
|
|
// trigger a restart loop.
|
|
func TestRenderDeterministicWithShuffledInput(t *testing.T) {
|
|
a := referenceInput()
|
|
b := referenceInput()
|
|
// shuffle b
|
|
b.Subnets[0], b.Subnets[5] = b.Subnets[5], b.Subnets[0]
|
|
b.ClientClasses[0], b.ClientClasses[1] = b.ClientClasses[1], b.ClientClasses[0]
|
|
b.Peers[0], b.Peers[1] = b.Peers[1], b.Peers[0]
|
|
|
|
oa, err := RenderDHCP4(a)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ob, err := RenderDHCP4(b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if oa != ob {
|
|
t.Errorf("render not deterministic under shuffled input\n--A--\n%s\n--B--\n%s", oa, ob)
|
|
}
|
|
}
|
|
|
|
// TestNoPodIPsInConfig guards the restart-loop invariant: the rendered config
|
|
// (which drives the config hash) must contain only stable DNS peer names.
|
|
func TestNoPodIPsInConfig(t *testing.T) {
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, ip := range []string{"10.", "172.", "192.168."} {
|
|
if strings.Contains(out, `"url": "http://`+ip) {
|
|
t.Errorf("pod IP leaked into HA peer url (contains %q)", ip)
|
|
}
|
|
}
|
|
if !strings.Contains(out, "pxe-headless") {
|
|
t.Errorf("expected stable headless DNS peer url")
|
|
}
|
|
}
|
|
|
|
func TestSubnetIDAssignmentStableAndUnique(t *testing.T) {
|
|
in := referenceInput()
|
|
out, err := RenderDHCP4(in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var root dhcp4Root
|
|
if err := json.Unmarshal([]byte(out), &root); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seen := map[int]bool{}
|
|
for _, s := range root.Dhcp4.Subnet4 {
|
|
if s.ID <= 0 {
|
|
t.Errorf("subnet %s has invalid id %d", s.Subnet, s.ID)
|
|
}
|
|
if seen[s.ID] {
|
|
t.Errorf("duplicate subnet id %d", s.ID)
|
|
}
|
|
seen[s.ID] = true
|
|
}
|
|
if len(seen) != 6 {
|
|
t.Errorf("expected 6 unique subnet ids, got %d", len(seen))
|
|
}
|
|
}
|
|
|
|
func TestExplicitSubnetIDPreserved(t *testing.T) {
|
|
in := referenceInput()
|
|
in.Subnets[2].Spec.ID = 42
|
|
out, err := RenderDHCP4(in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(out, `"id": 42`) {
|
|
t.Errorf("explicit subnet id 42 not preserved")
|
|
}
|
|
}
|
|
|
|
func TestRenderCtrlAgent(t *testing.T) {
|
|
out, err := RenderCtrlAgent()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var root map[string]any
|
|
if err := json.Unmarshal([]byte(out), &root); err != nil {
|
|
t.Fatalf("ctrl-agent config not valid JSON: %v", err)
|
|
}
|
|
for _, m := range []string{`"http-port": 8000`, `kea4-ctrl-socket`, `"dhcp4"`} {
|
|
if !strings.Contains(out, m) {
|
|
t.Errorf("ctrl-agent config missing %q", m)
|
|
}
|
|
}
|
|
// The HA hook runs with the dedicated listener disabled, so inbound HA traffic
|
|
// is served by the ctrl-agent; it must bind a pod-local address (0.0.0.0), not
|
|
// a ClusterIP, so peers reaching the per-pod Service's ClusterIP:8000 land here.
|
|
if !strings.Contains(out, `"http-host": "0.0.0.0"`) {
|
|
t.Errorf("ctrl-agent must bind 0.0.0.0 (pod-local) for CA-mediated HA, got: %s", out)
|
|
}
|
|
// Kea 2.6+ only accepts unix socket paths under /var/run/kea (exact string).
|
|
if !strings.Contains(out, `"socket-name": "/var/run/kea/`) {
|
|
t.Errorf("ctrl-agent socket-name must be under /var/run/kea, got: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestInitScriptHardensSocketDir(t *testing.T) {
|
|
// Kea 2.6+ rejects a socket dir "more relaxed than 750"; the emptyDir mount
|
|
// defaults to 0777, so the initContainer must chmod it (via $RUN_DIR) before
|
|
// the main containers exec kea.
|
|
if s := InitScript(); !strings.Contains(s, `chmod 0750 "$RUN_DIR"`) {
|
|
t.Errorf("init script must chmod 0750 the run dir, got:\n%s", s)
|
|
}
|
|
}
|
|
|
|
func TestInitScriptWaitsForConfigToValidate(t *testing.T) {
|
|
// The init script must gate startup on `kea-dhcp4 -t` so a cold-start HA peer
|
|
// DNS resolution failure retries (bounded) instead of crash-looping, and must
|
|
// fail loud after the cap so the kubelet restarts the initContainer.
|
|
s := InitScript()
|
|
for _, want := range []string{
|
|
`until "$DHCP4_BIN" -t "$DHCP4_CONF"`,
|
|
`"$i" -ge "$WAIT_ATTEMPTS"`,
|
|
"exit 1",
|
|
} {
|
|
if !strings.Contains(s, want) {
|
|
t.Errorf("init script must contain %q, got:\n%s", want, s)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInitScriptIsInterpolationFree(t *testing.T) {
|
|
// The whole point of the refactor: the script is a static embedded file, not
|
|
// a Go-interpolated string. It must carry no fmt verbs and must derive the
|
|
// HA peer name from the pod ordinal via $POD_NAME.
|
|
s := InitScript()
|
|
if strings.Contains(s, "%[") || strings.Contains(s, "%s") || strings.Contains(s, "%d") {
|
|
t.Errorf("init script must not contain fmt verbs:\n%s", s)
|
|
}
|
|
if !strings.Contains(s, `ord="${POD_NAME##*-}"`) {
|
|
t.Errorf("init script must derive the ordinal from $POD_NAME, got:\n%s", s)
|
|
}
|
|
}
|
|
|
|
func TestControlSocketPathAllowedByKea(t *testing.T) {
|
|
if !strings.HasPrefix(CtrlSocketPath, "/var/run/kea/") {
|
|
t.Errorf("CtrlSocketPath %q must live under /var/run/kea (kea 2.6+ restriction)", CtrlSocketPath)
|
|
}
|
|
out, err := RenderDHCP4(referenceInput())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(out, `"socket-name": "/var/run/kea/`) {
|
|
t.Errorf("dhcp4 control-socket must be under /var/run/kea, got: %s", out)
|
|
}
|
|
}
|