From 66ae5f5f3c16452b6d6c7bcb51e348cd7d31dcca Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sun, 9 Aug 2026 18:59:20 +1000 Subject: [PATCH] Point HA peer URLs at per-pod ClusterIP Services ## Why kea-dhcp4 crash-loops at HA hook load: kea 2.6's HA hook parses each peer url host as an IP literal and never resolves DNS, so the StatefulSet headless hostnames are rejected ("Failed to convert string to address ..."). Verified in-cluster that only an IP works (short name, FQDN both fail; `kea-dhcp4 -t` does not exercise this, which is why the v0.1.3 wait did not catch it). Pod IPs cannot be baked into the config because they change on restart and would roll-loop the StatefulSet via the config hash. ## How - create one ClusterIP Service per HA peer, selecting the pod by its statefulset.kubernetes.io/pod-name label, with publishNotReadyAddresses so peers are routable during bootstrap - render each HA peer url as its peer Service ClusterIP (a stable IP literal, safe in the config hash); reconcile Services before the ConfigMap and requeue until the ClusterIPs are allocated --- internal/controller/helpers.go | 18 ++++- internal/controller/keacluster_controller.go | 63 ++++++++++++--- internal/controller/keacluster_test.go | 85 ++++++++++++++++++-- 3 files changed, 149 insertions(+), 17 deletions(-) diff --git a/internal/controller/helpers.go b/internal/controller/helpers.go index 4327362..6fb6f0a 100644 --- a/internal/controller/helpers.go +++ b/internal/controller/helpers.go @@ -29,6 +29,10 @@ const ( clusterLabel = "kea.unkin.net/cluster" roleLabel = "kea.unkin.net/role" + // statefulSetPodNameLabel is stamped on every StatefulSet pod by Kubernetes; + // the per-pod ClusterIP Service selects a single pod through it. + statefulSetPodNameLabel = "statefulset.kubernetes.io/pod-name" + finalizer = "kea.unkin.net/finalizer" defaultOperatorImage = "git.unkin.net/unkin/kea-operator:latest" @@ -39,10 +43,20 @@ func headlessName(cluster string) string { return cluster + "-headless" } func serviceName(cluster string) string { return cluster } func configMapName(cluster string) string { return cluster + "-config" } func stsName(cluster string) string { return cluster } -func peerDNS(cluster, ns string, ordinal int) string { - return fmt.Sprintf("http://%s-%d.%s.%s:%d/", cluster, ordinal, headlessName(cluster), ns, 8000) + +// podName is the StatefulSet pod name for an ordinal. +func podName(cluster string, ordinal int) string { return fmt.Sprintf("%s-%d", cluster, ordinal) } + +// peerServiceName is the per-pod ClusterIP Service fronting one HA peer's +// ctrl-agent. Its stable ClusterIP is what the HA hook peer URL points at. +func peerServiceName(cluster string, ordinal int) string { + return fmt.Sprintf("%s-peer-%d", cluster, ordinal) } +// peerURL builds an HA peer control-channel URL. Kea 2.6's HA hook parses the +// host as an IP literal (no DNS), so this must be a stable IP address. +func peerURL(ip string) string { return fmt.Sprintf("http://%s:%d/", ip, 8000) } + func commonLabels(cluster string) map[string]string { return map[string]string{ managedByLabel: managedByValue, diff --git a/internal/controller/keacluster_controller.go b/internal/controller/keacluster_controller.go index d10ffb1..adae2b1 100644 --- a/internal/controller/keacluster_controller.go +++ b/internal/controller/keacluster_controller.go @@ -43,12 +43,15 @@ func (r *KeaClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - if err := r.reconcileConfigMap(ctx, &cluster); err != nil { - return r.fail(ctx, &cluster, "ConfigError", err) - } + // Services first: the per-pod ClusterIP Services must exist (and have their + // ClusterIPs allocated) before the ConfigMap is rendered, because the HA + // peer URLs baked into kea-dhcp4.conf are those stable ClusterIPs. if err := r.reconcileServices(ctx, &cluster); err != nil { return r.fail(ctx, &cluster, "ServiceError", err) } + if err := r.reconcileConfigMap(ctx, &cluster); err != nil { + return r.fail(ctx, &cluster, "ConfigError", err) + } sts, err := r.reconcileStatefulSet(ctx, &cluster) if err != nil { return r.fail(ctx, &cluster, "WorkloadError", err) @@ -118,16 +121,24 @@ func (r *KeaClusterReconciler) buildInput(ctx context.Context, c *v1alpha1.KeaCl } } + peers, err := r.peers(ctx, c) + if err != nil { + return kea.RenderInput{}, err + } return kea.RenderInput{ Cluster: *c, Subnets: subnets, ClientClasses: classes, - Peers: r.peers(c), + Peers: peers, }, nil } -// peers returns stable HA peer identities (DNS only, no pod IPs). -func (r *KeaClusterReconciler) peers(c *v1alpha1.KeaCluster) []kea.Peer { +// peers builds the HA peer list. Each URL points at the peer's per-pod ClusterIP +// Service address (a stable IP literal): kea 2.6's HA hook parses the peer URL +// host as an IP and never resolves DNS, so hostnames are rejected. Using the +// stable ClusterIP (not the pod IP) also keeps the config hash stable across +// pod restarts, so the StatefulSet does not roll-loop. +func (r *KeaClusterReconciler) peers(ctx context.Context, c *v1alpha1.KeaCluster) ([]kea.Peer, error) { replicas := int32(1) if c.Spec.Replicas != nil { replicas = *c.Spec.Replicas @@ -147,13 +158,22 @@ func (r *KeaClusterReconciler) peers(c *v1alpha1.KeaCluster) []kea.Peer { case i == 1: role = "secondary" } + svcName := peerServiceName(c.Name, int(i)) + var svc corev1.Service + if err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: svcName}, &svc); err != nil { + return nil, fmt.Errorf("peer service %s: %w", svcName, err) + } + ip := svc.Spec.ClusterIP + if ip == "" || ip == corev1.ClusterIPNone { + return nil, fmt.Errorf("peer service %s has no ClusterIP allocated yet", svcName) + } peers = append(peers, kea.Peer{ Name: fmt.Sprintf("server%d", i), - URL: peerDNS(c.Name, c.Namespace, int(i)), + URL: peerURL(ip), Role: role, }) } - return peers + return peers, nil } func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alpha1.KeaCluster) error { @@ -184,7 +204,7 @@ func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alph } func (r *KeaClusterReconciler) reconcileServices(ctx context.Context, c *v1alpha1.KeaCluster) error { - // Headless service for stable per-pod DNS (HA peer URLs, ctrl-agent). + // Headless service for stable per-pod DNS (ctrl-agent discovery). headless := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: headlessName(c.Name), Namespace: c.Namespace}} if _, err := ctrl.CreateOrUpdate(ctx, r.Client, headless, func() error { headless.Labels = commonLabels(c.Name) @@ -199,6 +219,31 @@ func (r *KeaClusterReconciler) reconcileServices(ctx context.Context, c *v1alpha return err } + // Per-pod ClusterIP Services: one stable IP per HA peer. Kea's HA hook needs + // an IP literal for each peer URL, and a ClusterIP survives pod restarts, so + // it is safe to bake into the (roll-triggering) config hash. Not-ready + // addresses are published so peers are routable during HA bootstrap. + replicas := int32(1) + if c.Spec.Replicas != nil { + replicas = *c.Spec.Replicas + } + for i := int32(0); i < replicas; i++ { + ord := int(i) + peerSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: peerServiceName(c.Name, ord), Namespace: c.Namespace}} + if _, err := ctrl.CreateOrUpdate(ctx, r.Client, peerSvc, func() error { + peerSvc.Labels = commonLabels(c.Name) + peerSvc.Spec.Type = corev1.ServiceTypeClusterIP + peerSvc.Spec.PublishNotReadyAddresses = true + peerSvc.Spec.Selector = map[string]string{statefulSetPodNameLabel: podName(c.Name, ord)} + peerSvc.Spec.Ports = []corev1.ServicePort{ + {Name: "ctrl", Port: kea.CtrlAgentPort, TargetPort: intstrFromInt(kea.CtrlAgentPort), Protocol: corev1.ProtocolTCP}, + } + return ctrl.SetControllerReference(c, peerSvc, r.Scheme) + }); err != nil { + return err + } + } + // Anycast DHCP service (LoadBalancer via PureLB by default). svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName(c.Name), Namespace: c.Namespace}} _, err := ctrl.CreateOrUpdate(ctx, r.Client, svc, func() error { diff --git a/internal/controller/keacluster_test.go b/internal/controller/keacluster_test.go index 0681cd3..d1afe07 100644 --- a/internal/controller/keacluster_test.go +++ b/internal/controller/keacluster_test.go @@ -40,6 +40,79 @@ func newClusterFixture() *v1alpha1.KeaCluster { } } +// peerServiceFixtures stands in for the per-pod ClusterIP Services with the +// ClusterIPs the apiserver would allocate (the fake client does not allocate), +// so peers() can read them when rendering the HA config. +func peerServiceFixtures() []client.Object { + return []client.Object{ + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-0", Namespace: "dhcp-system"}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.10"}, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-1", Namespace: "dhcp-system"}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.11"}, + }, + } +} + +func withPeers(objs ...client.Object) []client.Object { + return append(objs, peerServiceFixtures()...) +} + +// TestPeerURLsUseClusterIPs is the regression guard for the HA bootstrap fix: +// kea 2.6's HA hook rejects hostnames, so the rendered peer URLs must be the +// per-pod ClusterIP literals, never the headless DNS names. +func TestPeerURLsUseClusterIPs(t *testing.T) { + scheme := testScheme(t) + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&v1alpha1.KeaCluster{}). + WithObjects(withPeers(newClusterFixture())...). + Build() + r := &KeaClusterReconciler{Client: cl, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil { + t.Fatalf("reconcile: %v", err) + } + var cm corev1.ConfigMap + if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil { + t.Fatalf("configmap not created: %v", err) + } + conf := cm.Data["kea-dhcp4.conf"] + for _, want := range []string{"http://10.96.10.10:8000/", "http://10.96.10.11:8000/"} { + if !contains(conf, want) { + t.Errorf("HA peer URL %q missing from rendered config", want) + } + } + if contains(conf, "kea-headless") { + t.Errorf("HA peer URLs must not use headless DNS names (kea's HA hook rejects hostnames)") + } + + // The per-pod ClusterIP Services must exist. + for _, name := range []string{"pxe-peer-0", "pxe-peer-1"} { + var svc corev1.Service + if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil { + t.Errorf("per-pod service %s not present: %v", name, err) + } + } +} + +// TestPeersRequeueWithoutClusterIP proves the render blocks (returns an error to +// requeue) until the peer ClusterIPs are allocated, rather than emitting a +// hostname the HA hook would reject. +func TestPeersRequeueWithoutClusterIP(t *testing.T) { + scheme := testScheme(t) + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&v1alpha1.KeaCluster{}). + WithObjects(newClusterFixture()). + Build() + r := &KeaClusterReconciler{Client: cl, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err == nil { + t.Fatal("expected reconcile to error while peer ClusterIPs are unallocated") + } +} + func TestKeaClusterReconcileCreatesWorkload(t *testing.T) { scheme := testScheme(t) cluster := newClusterFixture() @@ -50,7 +123,7 @@ func TestKeaClusterReconcileCreatesWorkload(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&v1alpha1.KeaCluster{}). - WithObjects(cluster, subnet). + WithObjects(withPeers(cluster, subnet)...). Build() r := &KeaClusterReconciler{Client: cl, Scheme: scheme} @@ -82,8 +155,8 @@ func TestKeaClusterReconcileCreatesWorkload(t *testing.T) { t.Errorf("missing config-hash annotation") } - // Anycast + headless services. - for _, name := range []string{"pxe", "pxe-headless"} { + // Anycast + headless + per-pod peer services. + for _, name := range []string{"pxe", "pxe-headless", "pxe-peer-0", "pxe-peer-1"} { var svc corev1.Service if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil { t.Errorf("service %s not created: %v", name, err) @@ -102,7 +175,7 @@ func TestReconcileWiresInitContainer(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&v1alpha1.KeaCluster{}). - WithObjects(cluster). + WithObjects(withPeers(cluster)...). Build() r := &KeaClusterReconciler{Client: cl, Scheme: scheme} if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil { @@ -186,7 +259,7 @@ func TestConfigHashChangesWithSubnets(t *testing.T) { scheme := testScheme(t) hashFor := func(objs ...client.Object) string { - base := []client.Object{newClusterFixture()} + base := withPeers(newClusterFixture()) cl := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&v1alpha1.KeaCluster{}). @@ -226,7 +299,7 @@ func TestClusterRefFiltersSubnets(t *testing.T) { } cl := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&v1alpha1.KeaCluster{}). - WithObjects(cluster, mine, other).Build() + WithObjects(withPeers(cluster, mine, other)...).Build() r := &KeaClusterReconciler{Client: cl, Scheme: scheme} if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil { t.Fatal(err)