unkinben
66ae5f5f3c
## Why
kea-dhcp4 crash-loops at HA hook load: kea 2.6's HA hook parses each peer url host as an IP literal and never resolves DNS, so the StatefulSet headless hostnames are rejected ("Failed to convert string to address ..."). Verified in-cluster that only an IP works (short name, FQDN both fail; `kea-dhcp4 -t` does not exercise this, which is why the v0.1.3 wait did not catch it). Pod IPs cannot be baked into the config because they change on restart and would roll-loop the StatefulSet via the config hash.
## How
- create one ClusterIP Service per HA peer, selecting the pod by its statefulset.kubernetes.io/pod-name label, with publishNotReadyAddresses so peers are routable during bootstrap
- render each HA peer url as its peer Service ClusterIP (a stable IP literal, safe in the config hash); reconcile Services before the ConfigMap and requeue until the ClusterIPs are allocated
330 lines
12 KiB
Go
330 lines
12 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
|
|
)
|
|
|
|
func testScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
if err := clientgoscheme.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := v1alpha1.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func newClusterFixture() *v1alpha1.KeaCluster {
|
|
return &v1alpha1.KeaCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaClusterSpec{
|
|
Replicas: int32ptr(2),
|
|
DomainName: "main.unkin.net",
|
|
HA: v1alpha1.HASpec{Mode: v1alpha1.HAHotStandby},
|
|
},
|
|
}
|
|
}
|
|
|
|
// peerServiceFixtures stands in for the per-pod ClusterIP Services with the
|
|
// ClusterIPs the apiserver would allocate (the fake client does not allocate),
|
|
// so peers() can read them when rendering the HA config.
|
|
func peerServiceFixtures() []client.Object {
|
|
return []client.Object{
|
|
&corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-0", Namespace: "dhcp-system"},
|
|
Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.10"},
|
|
},
|
|
&corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-1", Namespace: "dhcp-system"},
|
|
Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.11"},
|
|
},
|
|
}
|
|
}
|
|
|
|
func withPeers(objs ...client.Object) []client.Object {
|
|
return append(objs, peerServiceFixtures()...)
|
|
}
|
|
|
|
// TestPeerURLsUseClusterIPs is the regression guard for the HA bootstrap fix:
|
|
// kea 2.6's HA hook rejects hostnames, so the rendered peer URLs must be the
|
|
// per-pod ClusterIP literals, never the headless DNS names.
|
|
func TestPeerURLsUseClusterIPs(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(newClusterFixture())...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
conf := cm.Data["kea-dhcp4.conf"]
|
|
for _, want := range []string{"http://10.96.10.10:8000/", "http://10.96.10.11:8000/"} {
|
|
if !contains(conf, want) {
|
|
t.Errorf("HA peer URL %q missing from rendered config", want)
|
|
}
|
|
}
|
|
if contains(conf, "kea-headless") {
|
|
t.Errorf("HA peer URLs must not use headless DNS names (kea's HA hook rejects hostnames)")
|
|
}
|
|
|
|
// The per-pod ClusterIP Services must exist.
|
|
for _, name := range []string{"pxe-peer-0", "pxe-peer-1"} {
|
|
var svc corev1.Service
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Errorf("per-pod service %s not present: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestPeersRequeueWithoutClusterIP proves the render blocks (returns an error to
|
|
// requeue) until the peer ClusterIPs are allocated, rather than emitting a
|
|
// hostname the HA hook would reject.
|
|
func TestPeersRequeueWithoutClusterIP(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(newClusterFixture()).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err == nil {
|
|
t.Fatal("expected reconcile to error while peer ClusterIPs are unallocated")
|
|
}
|
|
}
|
|
|
|
func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
subnet := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", Pools: []string{"198.18.13.200 - 198.18.13.220"}},
|
|
}
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster, subnet)...).
|
|
Build()
|
|
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
|
|
// ConfigMap rendered with the subnet.
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
if got := cm.Data["kea-dhcp4.conf"]; got == "" || !contains(got, "198.18.13.0/24") {
|
|
t.Errorf("configmap missing subnet render")
|
|
}
|
|
|
|
// StatefulSet with 2 containers and a config-hash annotation.
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatalf("statefulset not created: %v", err)
|
|
}
|
|
if *sts.Spec.Replicas != 2 {
|
|
t.Errorf("expected 2 replicas, got %d", *sts.Spec.Replicas)
|
|
}
|
|
if len(sts.Spec.Template.Spec.Containers) != 2 {
|
|
t.Errorf("expected kea-dhcp4 + kea-ctrl-agent containers, got %d", len(sts.Spec.Template.Spec.Containers))
|
|
}
|
|
if sts.Spec.Template.Annotations["kea.unkin.net/config-hash"] == "" {
|
|
t.Errorf("missing config-hash annotation")
|
|
}
|
|
|
|
// Anycast + headless + per-pod peer services.
|
|
for _, name := range []string{"pxe", "pxe-headless", "pxe-peer-0", "pxe-peer-1"} {
|
|
var svc corev1.Service
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Errorf("service %s not created: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestReconcileWiresInitContainer asserts the entrypoint refactor's pod shape:
|
|
// a kea-init initContainer prepares config + waits for HA peer DNS, and the two
|
|
// main containers exec kea directly with no wrapper shell. The init script comes
|
|
// from the ConfigMap (so its hash rolls the STS), and the old per-container
|
|
// entrypoint scripts are gone.
|
|
func TestReconcileWiresInitContainer(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
|
|
// ConfigMap carries init.sh and no longer the per-container entrypoints.
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
if cm.Data["init.sh"] == "" {
|
|
t.Errorf("configmap missing init.sh")
|
|
}
|
|
for _, gone := range []string{"entrypoint-dhcp4.sh", "entrypoint-ctrlagent.sh"} {
|
|
if _, ok := cm.Data[gone]; ok {
|
|
t.Errorf("configmap must not still carry %q", gone)
|
|
}
|
|
}
|
|
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatalf("statefulset not created: %v", err)
|
|
}
|
|
spec := sts.Spec.Template.Spec
|
|
|
|
// Exactly one initContainer, named kea-init, running init.sh with POD_NAME
|
|
// sourced from the downward API.
|
|
if len(spec.InitContainers) != 1 || spec.InitContainers[0].Name != "kea-init" {
|
|
t.Fatalf("expected one kea-init initContainer, got %#v", spec.InitContainers)
|
|
}
|
|
initC := spec.InitContainers[0]
|
|
if got := initC.Command; len(got) != 2 || got[0] != "/bin/sh" || got[1] != "/etc/kea-operator/init.sh" {
|
|
t.Errorf("init command = %v, want [/bin/sh /etc/kea-operator/init.sh]", got)
|
|
}
|
|
var podNameFromDownward bool
|
|
for _, e := range initC.Env {
|
|
if e.Name == "POD_NAME" && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == "metadata.name" {
|
|
podNameFromDownward = true
|
|
}
|
|
}
|
|
if !podNameFromDownward {
|
|
t.Errorf("init container must source POD_NAME from downward API metadata.name, env=%#v", initC.Env)
|
|
}
|
|
|
|
// Main containers exec kea directly (no /bin/sh wrapper).
|
|
wantCmd := map[string][]string{
|
|
"kea-dhcp4": {"/usr/sbin/kea-dhcp4", "-c", "/var/run/kea/kea-dhcp4.conf"},
|
|
"kea-ctrl-agent": {"/usr/sbin/kea-ctrl-agent", "-c", "/var/run/kea/kea-ctrl-agent.conf"},
|
|
}
|
|
for _, ctr := range spec.Containers {
|
|
want, ok := wantCmd[ctr.Name]
|
|
if !ok {
|
|
t.Errorf("unexpected container %q", ctr.Name)
|
|
continue
|
|
}
|
|
if len(ctr.Command) == 0 || ctr.Command[0] == "/bin/sh" {
|
|
t.Errorf("container %q must exec kea directly, got %v", ctr.Name, ctr.Command)
|
|
}
|
|
if !equalStrings(ctr.Command, want) {
|
|
t.Errorf("container %q command = %v, want %v", ctr.Name, ctr.Command, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func equalStrings(a, b []string) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// TestConfigHashChangesWithSubnets guards the roll trigger: adding a subnet
|
|
// must change the pod-template config hash (so the STS rolls).
|
|
func TestConfigHashChangesWithSubnets(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
|
|
hashFor := func(objs ...client.Object) string {
|
|
base := withPeers(newClusterFixture())
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(append(base, objs...)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sts.Spec.Template.Annotations["kea.unkin.net/config-hash"]
|
|
}
|
|
|
|
empty := hashFor()
|
|
withSubnet := hashFor(&v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24"},
|
|
})
|
|
if empty == withSubnet {
|
|
t.Errorf("config hash did not change when a subnet was added")
|
|
}
|
|
}
|
|
|
|
func TestClusterRefFiltersSubnets(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
mine := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "mine", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", ClusterRef: "pxe"},
|
|
}
|
|
other := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "10.9.9.0/24", ClusterRef: "someone-else"},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster, mine, other)...).Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cm corev1.ConfigMap
|
|
_ = cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm)
|
|
conf := cm.Data["kea-dhcp4.conf"]
|
|
if !contains(conf, "198.18.13.0/24") {
|
|
t.Errorf("cluster-matched subnet missing from config")
|
|
}
|
|
if contains(conf, "10.9.9.0/24") {
|
|
t.Errorf("subnet bound to another cluster leaked into config")
|
|
}
|
|
}
|
|
|
|
func contains(hay, needle string) bool {
|
|
return len(hay) >= len(needle) && (indexOf(hay, needle) >= 0)
|
|
}
|
|
|
|
func indexOf(hay, needle string) int {
|
|
for i := 0; i+len(needle) <= len(hay); i++ {
|
|
if hay[i:i+len(needle)] == needle {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|