Merge pull request 'Disable HA hook dedicated listener; route HA via ctrl-agent' (#8) from benvin/ha-listener-bind into main

Reviewed-on: #8

.woodpecker/pre-commit.yaml

@@ -17,3 +17,18 @@ steps:
     commands:
       - test -z "$(gofmt -l .)"
       - go vet ./...
+
+  - name: shellcheck
+    image: koalaman/shellcheck-alpine:stable
+    backend_options:
+      kubernetes:
+        serviceAccountName: kea-operator-ci
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: "500m"
+          limits:
+            memory: 512Mi
+            cpu: "1"
+    commands:
+      - shellcheck --shell=sh internal/kea/scripts/*.sh

README.md

@@ -35,9 +35,18 @@ Kea HA needs a **stable per-peer identity** (each server must know which peer it
 is, and peers reference each other by stable URL). That is exactly why this
 operator (like bind-operator) uses a **StatefulSet** rather than a bare
 Deployment: pods get stable ordinals (`<cluster>-0`, `<cluster>-1`) and headless
-DNS, the entrypoint derives `this-server-name` from the ordinal, and the peer
+DNS, an initContainer derives `this-server-name` from the ordinal, and the peer
 URLs are DNS names (never pod IPs, so the config hash never loops).
 
+Startup preconditions live in a `kea-init` initContainer (a committed,
+shellcheck-clean `internal/kea/scripts/init.sh` embedded via `go:embed` and
+parameterised by env vars — no shell is interpolated in Go). It hardens the
+shared run dir to `0750`, substitutes `this-server-name` from the pod ordinal,
+stages both configs into the shared `emptyDir`, and bounded-waits for the HA
+peer DNS to resolve (`kea-dhcp4 -t`, failing loud after the cap so the kubelet
+restarts it). The main `kea-dhcp4` and `kea-ctrl-agent` containers then exec kea
+directly with no wrapper shell.
+
 ### Anycast routing caveat (deployment follow-up)
 
 The DHCP `Service` is a `LoadBalancer` intended to receive a PureLB anycast IP;

internal/controller/helpers.go

@@ -29,6 +29,10 @@ const (
 	clusterLabel   = "kea.unkin.net/cluster"
 	roleLabel      = "kea.unkin.net/role"
 
+	// statefulSetPodNameLabel is stamped on every StatefulSet pod by Kubernetes;
+	// the per-pod ClusterIP Service selects a single pod through it.
+	statefulSetPodNameLabel = "statefulset.kubernetes.io/pod-name"
+
 	finalizer = "kea.unkin.net/finalizer"
 
 	defaultOperatorImage = "git.unkin.net/unkin/kea-operator:latest"
@@ -39,10 +43,20 @@ func headlessName(cluster string) string  { return cluster + "-headless" }
 func serviceName(cluster string) string   { return cluster }
 func configMapName(cluster string) string { return cluster + "-config" }
 func stsName(cluster string) string       { return cluster }
-func peerDNS(cluster, ns string, ordinal int) string {
+
-	return fmt.Sprintf("http://%s-%d.%s.%s:%d/", cluster, ordinal, headlessName(cluster), ns, 8000)
+// podName is the StatefulSet pod name for an ordinal.
+func podName(cluster string, ordinal int) string { return fmt.Sprintf("%s-%d", cluster, ordinal) }
+
+// peerServiceName is the per-pod ClusterIP Service fronting one HA peer's
+// ctrl-agent. Its stable ClusterIP is what the HA hook peer URL points at.
+func peerServiceName(cluster string, ordinal int) string {
+	return fmt.Sprintf("%s-peer-%d", cluster, ordinal)
 }
 
+// peerURL builds an HA peer control-channel URL. Kea 2.6's HA hook parses the
+// host as an IP literal (no DNS), so this must be a stable IP address.
+func peerURL(ip string) string { return fmt.Sprintf("http://%s:%d/", ip, 8000) }
+
 func commonLabels(cluster string) map[string]string {
 	return map[string]string{
 		managedByLabel: managedByValue,

internal/controller/keacluster_controller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -42,12 +43,15 @@ func (r *KeaClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	if err := r.reconcileConfigMap(ctx, &cluster); err != nil {
+	// Services first: the per-pod ClusterIP Services must exist (and have their
-		return r.fail(ctx, &cluster, "ConfigError", err)
+	// ClusterIPs allocated) before the ConfigMap is rendered, because the HA
-	}
+	// peer URLs baked into kea-dhcp4.conf are those stable ClusterIPs.
 	if err := r.reconcileServices(ctx, &cluster); err != nil {
 		return r.fail(ctx, &cluster, "ServiceError", err)
 	}
+	if err := r.reconcileConfigMap(ctx, &cluster); err != nil {
+		return r.fail(ctx, &cluster, "ConfigError", err)
+	}
 	sts, err := r.reconcileStatefulSet(ctx, &cluster)
 	if err != nil {
 		return r.fail(ctx, &cluster, "WorkloadError", err)
@@ -117,16 +121,24 @@ func (r *KeaClusterReconciler) buildInput(ctx context.Context, c *v1alpha1.KeaCl
 		}
 	}
 
+	peers, err := r.peers(ctx, c)
+	if err != nil {
+		return kea.RenderInput{}, err
+	}
 	return kea.RenderInput{
 		Cluster:       *c,
 		Subnets:       subnets,
 		ClientClasses: classes,
-		Peers:         r.peers(c),
+		Peers:         peers,
 	}, nil
 }
 
-// peers returns stable HA peer identities (DNS only, no pod IPs).
+// peers builds the HA peer list. Each URL points at the peer's per-pod ClusterIP
-func (r *KeaClusterReconciler) peers(c *v1alpha1.KeaCluster) []kea.Peer {
+// Service address (a stable IP literal): kea 2.6's HA hook parses the peer URL
+// host as an IP and never resolves DNS, so hostnames are rejected. Using the
+// stable ClusterIP (not the pod IP) also keeps the config hash stable across
+// pod restarts, so the StatefulSet does not roll-loop.
+func (r *KeaClusterReconciler) peers(ctx context.Context, c *v1alpha1.KeaCluster) ([]kea.Peer, error) {
 	replicas := int32(1)
 	if c.Spec.Replicas != nil {
 		replicas = *c.Spec.Replicas
@@ -146,13 +158,22 @@ func (r *KeaClusterReconciler) peers(c *v1alpha1.KeaCluster) []kea.Peer {
 		case i == 1:
 			role = "secondary"
 		}
+		svcName := peerServiceName(c.Name, int(i))
+		var svc corev1.Service
+		if err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: svcName}, &svc); err != nil {
+			return nil, fmt.Errorf("peer service %s: %w", svcName, err)
+		}
+		ip := svc.Spec.ClusterIP
+		if ip == "" || ip == corev1.ClusterIPNone {
+			return nil, fmt.Errorf("peer service %s has no ClusterIP allocated yet", svcName)
+		}
 		peers = append(peers, kea.Peer{
 			Name: fmt.Sprintf("server%d", i),
-			URL:  peerDNS(c.Name, c.Namespace, int(i)),
+			URL:  peerURL(ip),
 			Role: role,
 		})
 	}
-	return peers
+	return peers, nil
 }
 
 func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alpha1.KeaCluster) error {
@@ -175,8 +196,7 @@ func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alph
 		cm.Data = map[string]string{
 			"kea-dhcp4.conf":      dhcp4,
 			"kea-ctrl-agent.conf": agent,
-			"entrypoint-dhcp4.sh":     kea.EntrypointDHCP4(),
+			"init.sh":             kea.InitScript(),
-			"entrypoint-ctrlagent.sh": kea.EntrypointCtrlAgent(),
 		}
 		return ctrl.SetControllerReference(c, cm, r.Scheme)
 	})
@@ -184,7 +204,7 @@ func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alph
 }
 
 func (r *KeaClusterReconciler) reconcileServices(ctx context.Context, c *v1alpha1.KeaCluster) error {
-	// Headless service for stable per-pod DNS (HA peer URLs, ctrl-agent).
+	// Headless service for stable per-pod DNS (ctrl-agent discovery).
 	headless := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: headlessName(c.Name), Namespace: c.Namespace}}
 	if _, err := ctrl.CreateOrUpdate(ctx, r.Client, headless, func() error {
 		headless.Labels = commonLabels(c.Name)
@@ -199,6 +219,31 @@ func (r *KeaClusterReconciler) reconcileServices(ctx context.Context, c *v1alpha
 		return err
 	}
 
+	// Per-pod ClusterIP Services: one stable IP per HA peer. Kea's HA hook needs
+	// an IP literal for each peer URL, and a ClusterIP survives pod restarts, so
+	// it is safe to bake into the (roll-triggering) config hash. Not-ready
+	// addresses are published so peers are routable during HA bootstrap.
+	replicas := int32(1)
+	if c.Spec.Replicas != nil {
+		replicas = *c.Spec.Replicas
+	}
+	for i := int32(0); i < replicas; i++ {
+		ord := int(i)
+		peerSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: peerServiceName(c.Name, ord), Namespace: c.Namespace}}
+		if _, err := ctrl.CreateOrUpdate(ctx, r.Client, peerSvc, func() error {
+			peerSvc.Labels = commonLabels(c.Name)
+			peerSvc.Spec.Type = corev1.ServiceTypeClusterIP
+			peerSvc.Spec.PublishNotReadyAddresses = true
+			peerSvc.Spec.Selector = map[string]string{statefulSetPodNameLabel: podName(c.Name, ord)}
+			peerSvc.Spec.Ports = []corev1.ServicePort{
+				{Name: "ctrl", Port: kea.CtrlAgentPort, TargetPort: intstrFromInt(kea.CtrlAgentPort), Protocol: corev1.ProtocolTCP},
+			}
+			return ctrl.SetControllerReference(c, peerSvc, r.Scheme)
+		}); err != nil {
+			return err
+		}
+	}
+
 	// Anycast DHCP service (LoadBalancer via PureLB by default).
 	svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName(c.Name), Namespace: c.Namespace}}
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, svc, func() error {
@@ -288,10 +333,21 @@ func (r *KeaClusterReconciler) podTemplate(c *v1alpha1.KeaCluster, image, hash s
 		{Name: "run", MountPath: kea.RunDir},
 	}
 
+	// The initContainer finalises the per-pod config and bounded-waits for HA
+	// peer DNS; the main containers then exec kea directly with no wrapper shell.
+	initC := corev1.Container{
+		Name:         kea.ContainerInit,
+		Image:        image,
+		Command:      []string{"/bin/sh", kea.InitScriptPath},
+		Env:          initEnv(),
+		Resources:    c.Spec.Resources,
+		VolumeMounts: mounts,
+	}
+
 	dhcp4 := corev1.Container{
 		Name:      kea.ContainerDHCP4,
 		Image:     image,
-		Command:   []string{"/bin/sh", kea.ConfigDir + "/entrypoint-dhcp4.sh"},
+		Command:   []string{kea.DHCP4Bin, "-c", kea.DHCP4ConfPath},
 		Resources: c.Spec.Resources,
 		Ports: []corev1.ContainerPort{
 			{Name: "dhcp", ContainerPort: kea.DHCP4Port, Protocol: corev1.ProtocolUDP},
@@ -301,7 +357,7 @@ func (r *KeaClusterReconciler) podTemplate(c *v1alpha1.KeaCluster, image, hash s
 	agent := corev1.Container{
 		Name:      kea.ContainerCtrlAgent,
 		Image:     image,
-		Command:   []string{"/bin/sh", kea.ConfigDir + "/entrypoint-ctrlagent.sh"},
+		Command:   []string{kea.CtrlAgentBin, "-c", kea.CtrlAgentConfPath},
 		Resources: c.Spec.Resources,
 		Ports: []corev1.ContainerPort{
 			{Name: "ctrl", ContainerPort: kea.CtrlAgentPort, Protocol: corev1.ProtocolTCP},
@@ -320,6 +376,7 @@ func (r *KeaClusterReconciler) podTemplate(c *v1alpha1.KeaCluster, image, hash s
 			Annotations: map[string]string{"kea.unkin.net/config-hash": hash},
 		},
 		Spec: corev1.PodSpec{
+			InitContainers: []corev1.Container{initC},
 			Containers:     []corev1.Container{dhcp4, agent},
 			Volumes:        []corev1.Volume{volProjected, volRun},
 			NodeSelector:   c.Spec.NodeSelector,
@@ -329,6 +386,25 @@ func (r *KeaClusterReconciler) podTemplate(c *v1alpha1.KeaCluster, image, hash s
 	}
 }
 
+// initEnv is the environment the embedded init.sh reads. Passing paths and
+// tunables as env vars (rather than interpolating them into the script text)
+// keeps init.sh static, committed and shellcheck-clean.
+func initEnv() []corev1.EnvVar {
+	return []corev1.EnvVar{
+		{Name: kea.EnvPodName, ValueFrom: &corev1.EnvVarSource{
+			FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
+		}},
+		{Name: kea.EnvRunDir, Value: kea.RunDir},
+		{Name: kea.EnvConfigDir, Value: kea.ConfigDir},
+		{Name: kea.EnvThisServerPlaceholder, Value: kea.ThisServerPlaceholder},
+		{Name: kea.EnvDHCP4Bin, Value: kea.DHCP4Bin},
+		{Name: kea.EnvDHCP4Conf, Value: kea.DHCP4ConfPath},
+		{Name: kea.EnvCtrlAgentConf, Value: kea.CtrlAgentConfPath},
+		{Name: kea.EnvWaitAttempts, Value: strconv.Itoa(kea.WaitAttempts)},
+		{Name: kea.EnvWaitSleep, Value: strconv.Itoa(kea.WaitSleepSeconds)},
+	}
+}
+
 // reloadReadyPods best-effort hot-reloads config on ready pods via the
 // ctrl-agent REST channel, analogous to bind-operator's rndc reconfig.
 func (r *KeaClusterReconciler) reloadReadyPods(ctx context.Context, c *v1alpha1.KeaCluster) {

internal/controller/keacluster_test.go

@@ -40,6 +40,79 @@ func newClusterFixture() *v1alpha1.KeaCluster {
 	}
 }
 
+// peerServiceFixtures stands in for the per-pod ClusterIP Services with the
+// ClusterIPs the apiserver would allocate (the fake client does not allocate),
+// so peers() can read them when rendering the HA config.
+func peerServiceFixtures() []client.Object {
+	return []client.Object{
+		&corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-0", Namespace: "dhcp-system"},
+			Spec:       corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.10"},
+		},
+		&corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-1", Namespace: "dhcp-system"},
+			Spec:       corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.11"},
+		},
+	}
+}
+
+func withPeers(objs ...client.Object) []client.Object {
+	return append(objs, peerServiceFixtures()...)
+}
+
+// TestPeerURLsUseClusterIPs is the regression guard for the HA bootstrap fix:
+// kea 2.6's HA hook rejects hostnames, so the rendered peer URLs must be the
+// per-pod ClusterIP literals, never the headless DNS names.
+func TestPeerURLsUseClusterIPs(t *testing.T) {
+	scheme := testScheme(t)
+	cl := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithStatusSubresource(&v1alpha1.KeaCluster{}).
+		WithObjects(withPeers(newClusterFixture())...).
+		Build()
+	r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
+	if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
+		t.Fatalf("reconcile: %v", err)
+	}
+	var cm corev1.ConfigMap
+	if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
+		t.Fatalf("configmap not created: %v", err)
+	}
+	conf := cm.Data["kea-dhcp4.conf"]
+	for _, want := range []string{"http://10.96.10.10:8000/", "http://10.96.10.11:8000/"} {
+		if !contains(conf, want) {
+			t.Errorf("HA peer URL %q missing from rendered config", want)
+		}
+	}
+	if contains(conf, "kea-headless") {
+		t.Errorf("HA peer URLs must not use headless DNS names (kea's HA hook rejects hostnames)")
+	}
+
+	// The per-pod ClusterIP Services must exist.
+	for _, name := range []string{"pxe-peer-0", "pxe-peer-1"} {
+		var svc corev1.Service
+		if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
+			t.Errorf("per-pod service %s not present: %v", name, err)
+		}
+	}
+}
+
+// TestPeersRequeueWithoutClusterIP proves the render blocks (returns an error to
+// requeue) until the peer ClusterIPs are allocated, rather than emitting a
+// hostname the HA hook would reject.
+func TestPeersRequeueWithoutClusterIP(t *testing.T) {
+	scheme := testScheme(t)
+	cl := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithStatusSubresource(&v1alpha1.KeaCluster{}).
+		WithObjects(newClusterFixture()).
+		Build()
+	r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
+	if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err == nil {
+		t.Fatal("expected reconcile to error while peer ClusterIPs are unallocated")
+	}
+}
+
 func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
 	scheme := testScheme(t)
 	cluster := newClusterFixture()
@@ -50,7 +123,7 @@ func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
 	cl := fake.NewClientBuilder().
 		WithScheme(scheme).
 		WithStatusSubresource(&v1alpha1.KeaCluster{}).
-		WithObjects(cluster, subnet).
+		WithObjects(withPeers(cluster, subnet)...).
 		Build()
 
 	r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
@@ -82,8 +155,8 @@ func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
 		t.Errorf("missing config-hash annotation")
 	}
 
-	// Anycast + headless services.
+	// Anycast + headless + per-pod peer services.
-	for _, name := range []string{"pxe", "pxe-headless"} {
+	for _, name := range []string{"pxe", "pxe-headless", "pxe-peer-0", "pxe-peer-1"} {
 		var svc corev1.Service
 		if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
 			t.Errorf("service %s not created: %v", name, err)
@@ -91,13 +164,102 @@ func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
 	}
 }
 
+// TestReconcileWiresInitContainer asserts the entrypoint refactor's pod shape:
+// a kea-init initContainer prepares config + waits for HA peer DNS, and the two
+// main containers exec kea directly with no wrapper shell. The init script comes
+// from the ConfigMap (so its hash rolls the STS), and the old per-container
+// entrypoint scripts are gone.
+func TestReconcileWiresInitContainer(t *testing.T) {
+	scheme := testScheme(t)
+	cluster := newClusterFixture()
+	cl := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithStatusSubresource(&v1alpha1.KeaCluster{}).
+		WithObjects(withPeers(cluster)...).
+		Build()
+	r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
+	if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
+		t.Fatalf("reconcile: %v", err)
+	}
+
+	// ConfigMap carries init.sh and no longer the per-container entrypoints.
+	var cm corev1.ConfigMap
+	if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
+		t.Fatalf("configmap not created: %v", err)
+	}
+	if cm.Data["init.sh"] == "" {
+		t.Errorf("configmap missing init.sh")
+	}
+	for _, gone := range []string{"entrypoint-dhcp4.sh", "entrypoint-ctrlagent.sh"} {
+		if _, ok := cm.Data[gone]; ok {
+			t.Errorf("configmap must not still carry %q", gone)
+		}
+	}
+
+	var sts appsv1.StatefulSet
+	if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
+		t.Fatalf("statefulset not created: %v", err)
+	}
+	spec := sts.Spec.Template.Spec
+
+	// Exactly one initContainer, named kea-init, running init.sh with POD_NAME
+	// sourced from the downward API.
+	if len(spec.InitContainers) != 1 || spec.InitContainers[0].Name != "kea-init" {
+		t.Fatalf("expected one kea-init initContainer, got %#v", spec.InitContainers)
+	}
+	initC := spec.InitContainers[0]
+	if got := initC.Command; len(got) != 2 || got[0] != "/bin/sh" || got[1] != "/etc/kea-operator/init.sh" {
+		t.Errorf("init command = %v, want [/bin/sh /etc/kea-operator/init.sh]", got)
+	}
+	var podNameFromDownward bool
+	for _, e := range initC.Env {
+		if e.Name == "POD_NAME" && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == "metadata.name" {
+			podNameFromDownward = true
+		}
+	}
+	if !podNameFromDownward {
+		t.Errorf("init container must source POD_NAME from downward API metadata.name, env=%#v", initC.Env)
+	}
+
+	// Main containers exec kea directly (no /bin/sh wrapper).
+	wantCmd := map[string][]string{
+		"kea-dhcp4":      {"/usr/sbin/kea-dhcp4", "-c", "/var/run/kea/kea-dhcp4.conf"},
+		"kea-ctrl-agent": {"/usr/sbin/kea-ctrl-agent", "-c", "/var/run/kea/kea-ctrl-agent.conf"},
+	}
+	for _, ctr := range spec.Containers {
+		want, ok := wantCmd[ctr.Name]
+		if !ok {
+			t.Errorf("unexpected container %q", ctr.Name)
+			continue
+		}
+		if len(ctr.Command) == 0 || ctr.Command[0] == "/bin/sh" {
+			t.Errorf("container %q must exec kea directly, got %v", ctr.Name, ctr.Command)
+		}
+		if !equalStrings(ctr.Command, want) {
+			t.Errorf("container %q command = %v, want %v", ctr.Name, ctr.Command, want)
+		}
+	}
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
 // TestConfigHashChangesWithSubnets guards the roll trigger: adding a subnet
 // must change the pod-template config hash (so the STS rolls).
 func TestConfigHashChangesWithSubnets(t *testing.T) {
 	scheme := testScheme(t)
 
 	hashFor := func(objs ...client.Object) string {
-		base := []client.Object{newClusterFixture()}
+		base := withPeers(newClusterFixture())
 		cl := fake.NewClientBuilder().
 			WithScheme(scheme).
 			WithStatusSubresource(&v1alpha1.KeaCluster{}).
@@ -137,7 +299,7 @@ func TestClusterRefFiltersSubnets(t *testing.T) {
 	}
 	cl := fake.NewClientBuilder().WithScheme(scheme).
 		WithStatusSubresource(&v1alpha1.KeaCluster{}).
-		WithObjects(cluster, mine, other).Build()
+		WithObjects(withPeers(cluster, mine, other)...).Build()
 	r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
 	if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
 		t.Fatal(err)

internal/kea/agent.go

@@ -1,6 +1,6 @@
 package kea
 
-import "fmt"
+import _ "embed"
 
 type ctrlAgentRoot struct {
 	ControlAgent controlAgent `json:"Control-agent"`
@@ -27,27 +27,15 @@ func RenderCtrlAgent() (string, error) {
 	}})
 }
 
-// EntrypointDHCP4 is the kea-dhcp4 container entrypoint. It derives this pod's
+// initScript is the initContainer entrypoint. It is a committed, shellcheck-clean
-// HA peer name from the StatefulSet ordinal, substitutes the placeholder in the
+// shell file (no fmt.Sprintf interpolation) parameterised entirely by the
-// projected config, and execs the server.
+// environment variables the operator sets on the initContainer. It prepares the
-func EntrypointDHCP4() string {
+// shared run dir, finalises this pod's kea-dhcp4 config from the StatefulSet
-	return fmt.Sprintf(`#!/bin/sh
+// ordinal, and bounded-waits for the HA peer DNS to resolve before the main
-set -e
+// kea-dhcp4 / kea-ctrl-agent containers exec kea directly.
-ORD="${HOSTNAME##*-}"
+//
-mkdir -p %[1]s
+//go:embed scripts/init.sh
-chmod 0750 %[1]s
+var initScript string
-sed "s/%[2]s/server${ORD}/g" %[3]s/kea-dhcp4.conf > %[4]s
-exec %[5]s -c %[4]s
-`, RunDir, ThisServerPlaceholder, ConfigDir, DHCP4ConfPath, DHCP4Bin)
-}
 
-// EntrypointCtrlAgent is the kea-ctrl-agent container entrypoint.
+// InitScript returns the initContainer entrypoint shell script.
-func EntrypointCtrlAgent() string {
+func InitScript() string { return initScript }
-	return fmt.Sprintf(`#!/bin/sh
-set -e
-mkdir -p %[1]s
-chmod 0750 %[1]s
-cp %[2]s/kea-ctrl-agent.conf %[3]s
-exec %[4]s -c %[3]s
-`, RunDir, ConfigDir, CtrlAgentConfPath, CtrlAgentBin)
-}

internal/kea/config.go

@@ -241,6 +241,16 @@ func hooks(in RenderInput) []hookLib {
 		"max-response-delay":  firstNonZero(ha.MaxResponseDelay, 60000),
 		"max-ack-delay":       firstNonZero(ha.MaxAckDelay, 5000),
 		"max-unacked-clients": firstNonZero(ha.MaxUnackedClients, 5),
+		// With core multi-threading enabled (Kea 2.6 default) the HA hook would
+		// open a dedicated HTTP listener bound to this server's peer url address —
+		// here a per-pod ClusterIP (virtual, kube-proxy DNAT) that is not
+		// assignable on the pod, so the bind fails. Disable it so inbound HA
+		// traffic flows via kea-ctrl-agent, which binds 0.0.0.0:CtrlAgentPort;
+		// peers stay reachable at their ClusterIP:CtrlAgentPort via the per-pod Service.
+		"multi-threading": map[string]any{
+			"enable-multi-threading":  true,
+			"http-dedicated-listener": false,
+		},
 		"peers": peers,
 	}
 	libs = append(libs, hookLib{

internal/kea/config_test.go

@@ -114,6 +114,53 @@ func TestRenderDHCP4ReferenceSemantics(t *testing.T) {
 	}
 }
 
+// TestHADedicatedListenerDisabled guards the HA listener/bind fix: with Kea 2.6
+// core multi-threading on by default, the HA hook would otherwise open a
+// dedicated HTTP listener bound to this server's peer url — a per-pod ClusterIP
+// that is virtual (kube-proxy DNAT) and unassignable on the pod, failing with
+// "Cannot assign requested address". The rendered config must disable the
+// dedicated listener so inbound HA traffic is served by kea-ctrl-agent
+// (0.0.0.0:8000) while peers stay reachable at their ClusterIP:8000.
+func TestHADedicatedListenerDisabled(t *testing.T) {
+	out, err := RenderDHCP4(referenceInput())
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+	if !strings.Contains(out, `"http-dedicated-listener": false`) {
+		t.Errorf("HA hook must disable the dedicated HTTP listener (it binds this server's ClusterIP peer url, unassignable on the pod); got:\n%s", out)
+	}
+	if !strings.Contains(out, `"enable-multi-threading": true`) {
+		t.Errorf("HA multi-threading must stay enabled (CA-mediated HA traffic); got:\n%s", out)
+	}
+
+	// The CA-mediated route only works because the dedicated listener stays off
+	// on the same port the ctrl-agent binds: parse out the HA hook and assert it.
+	var root dhcp4Root
+	if err := json.Unmarshal([]byte(out), &root); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	var haParams map[string]any
+	for _, h := range root.Dhcp4.HooksLibraries {
+		if strings.Contains(h.Library, "libdhcp_ha.so") {
+			haParams = h.Parameters
+		}
+	}
+	if haParams == nil {
+		t.Fatal("HA hook library not present in rendered config")
+	}
+	rels, ok := haParams["high-availability"].([]any)
+	if !ok || len(rels) != 1 {
+		t.Fatalf("high-availability block malformed: %#v", haParams["high-availability"])
+	}
+	mt, ok := rels[0].(map[string]any)["multi-threading"].(map[string]any)
+	if !ok {
+		t.Fatalf("HA relationship missing multi-threading block: %#v", rels[0])
+	}
+	if mt["http-dedicated-listener"] != false {
+		t.Errorf("http-dedicated-listener must be false, got %#v", mt["http-dedicated-listener"])
+	}
+}
+
 // TestSubnetWithoutPoolIsDeclared verifies the pool-less subnet still appears
 // (Kea must know the subnet to service relayed requests) but carries no pools.
 func TestSubnetWithoutPoolIsDeclared(t *testing.T) {
@@ -231,21 +278,53 @@ func TestRenderCtrlAgent(t *testing.T) {
 			t.Errorf("ctrl-agent config missing %q", m)
 		}
 	}
+	// The HA hook runs with the dedicated listener disabled, so inbound HA traffic
+	// is served by the ctrl-agent; it must bind a pod-local address (0.0.0.0), not
+	// a ClusterIP, so peers reaching the per-pod Service's ClusterIP:8000 land here.
+	if !strings.Contains(out, `"http-host": "0.0.0.0"`) {
+		t.Errorf("ctrl-agent must bind 0.0.0.0 (pod-local) for CA-mediated HA, got: %s", out)
+	}
 	// Kea 2.6+ only accepts unix socket paths under /var/run/kea (exact string).
 	if !strings.Contains(out, `"socket-name": "/var/run/kea/`) {
 		t.Errorf("ctrl-agent socket-name must be under /var/run/kea, got: %s", out)
 	}
 }
 
-func TestEntrypointsHardenSocketDir(t *testing.T) {
+func TestInitScriptHardensSocketDir(t *testing.T) {
 	// Kea 2.6+ rejects a socket dir "more relaxed than 750"; the emptyDir mount
-	// defaults to 0777, so the entrypoints must chmod it before exec'ing kea.
+	// defaults to 0777, so the initContainer must chmod it (via $RUN_DIR) before
-	want := "chmod 0750 " + RunDir
+	// the main containers exec kea.
-	if ep := EntrypointDHCP4(); !strings.Contains(ep, want) {
+	if s := InitScript(); !strings.Contains(s, `chmod 0750 "$RUN_DIR"`) {
-		t.Errorf("dhcp4 entrypoint must %q, got:\n%s", want, ep)
+		t.Errorf("init script must chmod 0750 the run dir, got:\n%s", s)
 	}
-	if ep := EntrypointCtrlAgent(); !strings.Contains(ep, want) {
+}
-		t.Errorf("ctrl-agent entrypoint must %q, got:\n%s", want, ep)
+
+func TestInitScriptWaitsForConfigToValidate(t *testing.T) {
+	// The init script must gate startup on `kea-dhcp4 -t` so a cold-start HA peer
+	// DNS resolution failure retries (bounded) instead of crash-looping, and must
+	// fail loud after the cap so the kubelet restarts the initContainer.
+	s := InitScript()
+	for _, want := range []string{
+		`until "$DHCP4_BIN" -t "$DHCP4_CONF"`,
+		`"$i" -ge "$WAIT_ATTEMPTS"`,
+		"exit 1",
+	} {
+		if !strings.Contains(s, want) {
+			t.Errorf("init script must contain %q, got:\n%s", want, s)
+		}
+	}
+}
+
+func TestInitScriptIsInterpolationFree(t *testing.T) {
+	// The whole point of the refactor: the script is a static embedded file, not
+	// a Go-interpolated string. It must carry no fmt verbs and must derive the
+	// HA peer name from the pod ordinal via $POD_NAME.
+	s := InitScript()
+	if strings.Contains(s, "%[") || strings.Contains(s, "%s") || strings.Contains(s, "%d") {
+		t.Errorf("init script must not contain fmt verbs:\n%s", s)
+	}
+	if !strings.Contains(s, `ord="${POD_NAME##*-}"`) {
+		t.Errorf("init script must derive the ordinal from $POD_NAME, got:\n%s", s)
 	}
 }
 

internal/kea/consts.go

@@ -7,6 +7,9 @@ const (
 	ContainerDHCP4 = "kea-dhcp4"
 	// ContainerCtrlAgent is the kea-ctrl-agent container name.
 	ContainerCtrlAgent = "kea-ctrl-agent"
+	// ContainerInit is the initContainer that finalises config and waits for
+	// HA peer DNS before the main containers start.
+	ContainerInit = "kea-init"
 
 	// ConfigDir is where projected config is mounted read-only.
 	ConfigDir = "/etc/kea-operator"
@@ -22,8 +25,9 @@ const (
 	CtrlAgentConfPath = RunDir + "/kea-ctrl-agent.conf"
 	// CtrlSocketPath is the unix control socket between ctrl-agent and dhcp4.
 	CtrlSocketPath = RunDir + "/kea4-ctrl-socket"
-	// EntrypointPath is the generated container entrypoint.
+	// InitScriptPath is where the initContainer entrypoint is projected from the
-	EntrypointPath = ConfigDir + "/entrypoint.sh"
+	// ConfigMap.
+	InitScriptPath = ConfigDir + "/init.sh"
 
 	// DHCP4Bin is the kea-dhcp4 server binary.
 	DHCP4Bin = "/usr/sbin/kea-dhcp4"
@@ -47,4 +51,35 @@ const (
 
 	// ClientArchOption is the DHCP option code carrying PXE client arch.
 	ClientArchOption = 93
+
+	// WaitAttempts caps the initContainer's bounded wait for the HA peer DNS to
+	// resolve (i.e. for the rendered config to pass "kea-dhcp4 -t").
+	WaitAttempts = 60
+	// WaitSleepSeconds is the delay between bounded-wait attempts.
+	WaitSleepSeconds = 2
+)
+
+// Environment variable names the operator sets on the initContainer. The
+// embedded init.sh reads only these; keeping the names here means the script
+// stays free of any Go string interpolation.
+const (
+	// EnvPodName carries the pod name (downward API metadata.name); its ordinal
+	// suffix selects this pod's HA peer name.
+	EnvPodName = "POD_NAME"
+	// EnvRunDir is the shared run/socket dir path.
+	EnvRunDir = "RUN_DIR"
+	// EnvConfigDir is the read-only projected config dir path.
+	EnvConfigDir = "CONFIG_DIR"
+	// EnvThisServerPlaceholder is the token replaced with this pod's HA peer name.
+	EnvThisServerPlaceholder = "THIS_SERVER_PLACEHOLDER"
+	// EnvDHCP4Bin is the kea-dhcp4 binary path (used for the -t config check).
+	EnvDHCP4Bin = "DHCP4_BIN"
+	// EnvDHCP4Conf is the finalized kea-dhcp4 config path in the shared run dir.
+	EnvDHCP4Conf = "DHCP4_CONF"
+	// EnvCtrlAgentConf is the staged kea-ctrl-agent config path in the run dir.
+	EnvCtrlAgentConf = "CTRL_AGENT_CONF"
+	// EnvWaitAttempts is the bounded-wait attempt cap.
+	EnvWaitAttempts = "WAIT_ATTEMPTS"
+	// EnvWaitSleep is the per-attempt sleep in seconds.
+	EnvWaitSleep = "WAIT_SLEEP"
 )

internal/kea/scripts/init.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# kea-operator initContainer.
+#
+# Prepares the shared run dir, finalises this pod's kea-dhcp4 config (the HA
+# this-server-name is derived from the StatefulSet ordinal, known only at pod
+# start), and waits for the HA peer DNS to resolve before the main kea
+# containers start. Every input arrives as an environment variable set by the
+# operator; nothing is interpolated into this file, so it is shellcheck-clean
+# and testable on its own.
+set -eu
+
+# StatefulSet pod names are "<sts>-<ordinal>"; the ordinal is this HA peer's id.
+ord="${POD_NAME##*-}"
+
+# The run dir is a shared emptyDir. Kea 2.6+ refuses a control socket in a
+# world-accessible directory, so tighten it to 0750.
+mkdir -p "$RUN_DIR"
+chmod 0750 "$RUN_DIR"
+
+# Finalise the per-pod dhcp4 config: substitute this pod's HA peer name into the
+# shared (pod-independent) config projected from the ConfigMap.
+sed "s/${THIS_SERVER_PLACEHOLDER}/server${ord}/g" \
+	"${CONFIG_DIR}/kea-dhcp4.conf" >"$DHCP4_CONF"
+
+# The ctrl-agent config is pod-independent; stage it in the shared run dir so
+# the main container can exec kea directly with no wrapper.
+cp "${CONFIG_DIR}/kea-ctrl-agent.conf" "$CTRL_AGENT_CONF"
+
+# The HA hook resolves peer URL hostnames once at load; on a cold start the
+# StatefulSet peer DNS records may not resolve yet and kea exits hard instead of
+# retrying. Wait (bounded) for the config to validate, then fail loud so the
+# kubelet restarts this initContainer rather than starting a doomed server.
+i=0
+until "$DHCP4_BIN" -t "$DHCP4_CONF" >/dev/null 2>&1; do
+	i=$((i + 1))
+	if [ "$i" -ge "$WAIT_ATTEMPTS" ]; then
+		echo "kea-init: config failed to validate after ${WAIT_ATTEMPTS} attempts" >&2
+		exit 1
+	fi
+	sleep "$WAIT_SLEEP"
+done