458 lines
16 KiB
Go
458 lines
16 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
|
|
)
|
|
|
|
func testScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
if err := clientgoscheme.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := v1alpha1.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func newClusterFixture() *v1alpha1.KeaCluster {
|
|
return &v1alpha1.KeaCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaClusterSpec{
|
|
Replicas: int32ptr(2),
|
|
DomainName: "main.unkin.net",
|
|
HA: v1alpha1.HASpec{Mode: v1alpha1.HAHotStandby},
|
|
},
|
|
}
|
|
}
|
|
|
|
// peerServiceFixtures stands in for the per-pod ClusterIP Services with the
|
|
// ClusterIPs the apiserver would allocate (the fake client does not allocate),
|
|
// so peers() can read them when rendering the HA config.
|
|
func peerServiceFixtures() []client.Object {
|
|
return []client.Object{
|
|
&corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-0", Namespace: "dhcp-system"},
|
|
Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.10"},
|
|
},
|
|
&corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "pxe-peer-1", Namespace: "dhcp-system"},
|
|
Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP, ClusterIP: "10.96.10.11"},
|
|
},
|
|
}
|
|
}
|
|
|
|
func withPeers(objs ...client.Object) []client.Object {
|
|
return append(objs, peerServiceFixtures()...)
|
|
}
|
|
|
|
// TestPeerURLsUseClusterIPs is the regression guard for the HA bootstrap fix:
|
|
// kea 2.6's HA hook rejects hostnames, so the rendered peer URLs must be the
|
|
// per-pod ClusterIP literals, never the headless DNS names.
|
|
func TestPeerURLsUseClusterIPs(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(newClusterFixture())...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
conf := cm.Data["kea-dhcp4.conf"]
|
|
for _, want := range []string{"http://10.96.10.10:8000/", "http://10.96.10.11:8000/"} {
|
|
if !contains(conf, want) {
|
|
t.Errorf("HA peer URL %q missing from rendered config", want)
|
|
}
|
|
}
|
|
if contains(conf, "kea-headless") {
|
|
t.Errorf("HA peer URLs must not use headless DNS names (kea's HA hook rejects hostnames)")
|
|
}
|
|
|
|
// The per-pod ClusterIP Services must exist.
|
|
for _, name := range []string{"pxe-peer-0", "pxe-peer-1"} {
|
|
var svc corev1.Service
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Errorf("per-pod service %s not present: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestPeersRequeueWithoutClusterIP proves the render blocks (returns an error to
|
|
// requeue) until the peer ClusterIPs are allocated, rather than emitting a
|
|
// hostname the HA hook would reject.
|
|
func TestPeersRequeueWithoutClusterIP(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(newClusterFixture()).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err == nil {
|
|
t.Fatal("expected reconcile to error while peer ClusterIPs are unallocated")
|
|
}
|
|
}
|
|
|
|
func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
subnet := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", Pools: []string{"198.18.13.200 - 198.18.13.220"}},
|
|
}
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster, subnet)...).
|
|
Build()
|
|
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
|
|
// ConfigMap rendered with the subnet.
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
if got := cm.Data["kea-dhcp4.conf"]; got == "" || !contains(got, "198.18.13.0/24") {
|
|
t.Errorf("configmap missing subnet render")
|
|
}
|
|
|
|
// StatefulSet with 2 containers and a config-hash annotation.
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatalf("statefulset not created: %v", err)
|
|
}
|
|
if *sts.Spec.Replicas != 2 {
|
|
t.Errorf("expected 2 replicas, got %d", *sts.Spec.Replicas)
|
|
}
|
|
if len(sts.Spec.Template.Spec.Containers) != 2 {
|
|
t.Errorf("expected kea-dhcp4 + kea-ctrl-agent containers, got %d", len(sts.Spec.Template.Spec.Containers))
|
|
}
|
|
if sts.Spec.Template.Annotations["kea.unkin.net/config-hash"] == "" {
|
|
t.Errorf("missing config-hash annotation")
|
|
}
|
|
|
|
// Anycast + headless + per-pod peer services.
|
|
for _, name := range []string{"pxe", "pxe-headless", "pxe-peer-0", "pxe-peer-1"} {
|
|
var svc corev1.Service
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Errorf("service %s not created: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestReconcileWiresInitContainer asserts the entrypoint refactor's pod shape:
|
|
// a kea-init initContainer prepares config + waits for HA peer DNS, and the two
|
|
// main containers exec kea directly with no wrapper shell. The init script comes
|
|
// from the ConfigMap (so its hash rolls the STS), and the old per-container
|
|
// entrypoint scripts are gone.
|
|
func TestReconcileWiresInitContainer(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
|
|
// ConfigMap carries init.sh and no longer the per-container entrypoints.
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
|
|
t.Fatalf("configmap not created: %v", err)
|
|
}
|
|
if cm.Data["init.sh"] == "" {
|
|
t.Errorf("configmap missing init.sh")
|
|
}
|
|
for _, gone := range []string{"entrypoint-dhcp4.sh", "entrypoint-ctrlagent.sh"} {
|
|
if _, ok := cm.Data[gone]; ok {
|
|
t.Errorf("configmap must not still carry %q", gone)
|
|
}
|
|
}
|
|
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatalf("statefulset not created: %v", err)
|
|
}
|
|
spec := sts.Spec.Template.Spec
|
|
|
|
// Exactly one initContainer, named kea-init, running init.sh with POD_NAME
|
|
// sourced from the downward API.
|
|
if len(spec.InitContainers) != 1 || spec.InitContainers[0].Name != "kea-init" {
|
|
t.Fatalf("expected one kea-init initContainer, got %#v", spec.InitContainers)
|
|
}
|
|
initC := spec.InitContainers[0]
|
|
if got := initC.Command; len(got) != 2 || got[0] != "/bin/sh" || got[1] != "/etc/kea-operator/init.sh" {
|
|
t.Errorf("init command = %v, want [/bin/sh /etc/kea-operator/init.sh]", got)
|
|
}
|
|
var podNameFromDownward bool
|
|
for _, e := range initC.Env {
|
|
if e.Name == "POD_NAME" && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == "metadata.name" {
|
|
podNameFromDownward = true
|
|
}
|
|
}
|
|
if !podNameFromDownward {
|
|
t.Errorf("init container must source POD_NAME from downward API metadata.name, env=%#v", initC.Env)
|
|
}
|
|
|
|
// Main containers exec kea directly (no /bin/sh wrapper).
|
|
wantCmd := map[string][]string{
|
|
"kea-dhcp4": {"/usr/sbin/kea-dhcp4", "-c", "/var/run/kea/kea-dhcp4.conf"},
|
|
"kea-ctrl-agent": {"/usr/sbin/kea-ctrl-agent", "-c", "/var/run/kea/kea-ctrl-agent.conf"},
|
|
}
|
|
for _, ctr := range spec.Containers {
|
|
want, ok := wantCmd[ctr.Name]
|
|
if !ok {
|
|
t.Errorf("unexpected container %q", ctr.Name)
|
|
continue
|
|
}
|
|
if len(ctr.Command) == 0 || ctr.Command[0] == "/bin/sh" {
|
|
t.Errorf("container %q must exec kea directly, got %v", ctr.Name, ctr.Command)
|
|
}
|
|
if !equalStrings(ctr.Command, want) {
|
|
t.Errorf("container %q command = %v, want %v", ctr.Name, ctr.Command, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func equalStrings(a, b []string) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// TestConfigHashChangesWithSubnets guards the roll trigger: adding a subnet
|
|
// must change the pod-template config hash (so the STS rolls).
|
|
func TestConfigHashChangesWithSubnets(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
|
|
hashFor := func(objs ...client.Object) string {
|
|
base := withPeers(newClusterFixture())
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(append(base, objs...)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var sts appsv1.StatefulSet
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sts.Spec.Template.Annotations["kea.unkin.net/config-hash"]
|
|
}
|
|
|
|
empty := hashFor()
|
|
withSubnet := hashFor(&v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24"},
|
|
})
|
|
if empty == withSubnet {
|
|
t.Errorf("config hash did not change when a subnet was added")
|
|
}
|
|
}
|
|
|
|
func TestClusterRefFiltersSubnets(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
mine := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "mine", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", ClusterRef: "pxe"},
|
|
}
|
|
other := &v1alpha1.KeaSubnet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "dhcp-system"},
|
|
Spec: v1alpha1.KeaSubnetSpec{Subnet: "10.9.9.0/24", ClusterRef: "someone-else"},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster, mine, other)...).Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cm corev1.ConfigMap
|
|
_ = cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm)
|
|
conf := cm.Data["kea-dhcp4.conf"]
|
|
if !contains(conf, "198.18.13.0/24") {
|
|
t.Errorf("cluster-matched subnet missing from config")
|
|
}
|
|
if contains(conf, "10.9.9.0/24") {
|
|
t.Errorf("subnet bound to another cluster leaked into config")
|
|
}
|
|
}
|
|
|
|
// TestSteadyStateReconcileLeavesServicesAlone guards the reconcile hot-loop:
|
|
// once the apiserver has defaulted the Services (NodePort, TargetPort) and
|
|
// another controller has annotated one, a no-change reconcile must not write
|
|
// any Service, since each write re-triggers the Owns(Service) watch.
|
|
func TestSteadyStateReconcileLeavesServicesAlone(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
cluster.Spec.Service.IPAddressPool = "dhcp"
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
ctx := context.Background()
|
|
req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatalf("first reconcile: %v", err)
|
|
}
|
|
|
|
names := []string{"pxe", "pxe-headless", "pxe-peer-0", "pxe-peer-1"}
|
|
const foreign = "field.cattle.io/publicEndpoints"
|
|
versions := map[string]string{}
|
|
for _, name := range names {
|
|
var svc corev1.Service
|
|
if err := cl.Get(ctx, types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if svc.Spec.Ports[0].TargetPort.IntVal == 0 {
|
|
svc.Spec.Ports[0].TargetPort = intstrFromInt(int(svc.Spec.Ports[0].Port))
|
|
}
|
|
if name == "pxe" {
|
|
svc.Spec.Ports[0].NodePort = 31067
|
|
if svc.Annotations == nil {
|
|
svc.Annotations = map[string]string{}
|
|
}
|
|
svc.Annotations[foreign] = "[]"
|
|
}
|
|
if err := cl.Update(ctx, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
versions[name] = svc.ResourceVersion
|
|
}
|
|
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatalf("second reconcile: %v", err)
|
|
}
|
|
for _, name := range names {
|
|
var svc corev1.Service
|
|
if err := cl.Get(ctx, types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if svc.ResourceVersion != versions[name] {
|
|
t.Errorf("service %s updated on a no-change reconcile", name)
|
|
}
|
|
if name != "pxe" {
|
|
continue
|
|
}
|
|
if svc.Annotations[foreign] != "[]" {
|
|
t.Errorf("foreign annotation %s dropped: %v", foreign, svc.Annotations)
|
|
}
|
|
if svc.Annotations["purelb.io/service-group"] != "dhcp" {
|
|
t.Errorf("operator annotation missing: %v", svc.Annotations)
|
|
}
|
|
if svc.Spec.Ports[0].NodePort != 31067 {
|
|
t.Errorf("allocated NodePort not preserved: %d", svc.Spec.Ports[0].NodePort)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRemovedServiceAnnotationIsDropped ensures annotations dropped from the
|
|
// spec are removed from the Service while foreign annotations survive.
|
|
func TestRemovedServiceAnnotationIsDropped(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := newClusterFixture()
|
|
cluster.Spec.Service.IPAddressPool = "dhcp"
|
|
cluster.Spec.Service.Annotations = map[string]string{"example.com/x": "1"}
|
|
cl := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithStatusSubresource(&v1alpha1.KeaCluster{}).
|
|
WithObjects(withPeers(cluster)...).
|
|
Build()
|
|
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
|
|
ctx := context.Background()
|
|
key := types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}
|
|
req := ctrl.Request{NamespacedName: key}
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatalf("first reconcile: %v", err)
|
|
}
|
|
|
|
const foreign = "field.cattle.io/publicEndpoints"
|
|
var svc corev1.Service
|
|
if err := cl.Get(ctx, key, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if svc.Annotations["example.com/x"] != "1" || svc.Annotations["purelb.io/service-group"] != "dhcp" {
|
|
t.Fatalf("operator annotations missing: %v", svc.Annotations)
|
|
}
|
|
svc.Annotations[foreign] = "[]"
|
|
if err := cl.Update(ctx, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var kc v1alpha1.KeaCluster
|
|
if err := cl.Get(ctx, key, &kc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kc.Spec.Service.Annotations = nil
|
|
kc.Spec.Service.IPAddressPool = ""
|
|
if err := cl.Update(ctx, &kc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatalf("second reconcile: %v", err)
|
|
}
|
|
if err := cl.Get(ctx, key, &svc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, k := range []string{"example.com/x", "purelb.io/service-group", managedAnnotationsKey} {
|
|
if _, ok := svc.Annotations[k]; ok {
|
|
t.Errorf("stale annotation %s kept: %v", k, svc.Annotations)
|
|
}
|
|
}
|
|
if svc.Annotations[foreign] != "[]" {
|
|
t.Errorf("foreign annotation dropped: %v", svc.Annotations)
|
|
}
|
|
}
|
|
|
|
func contains(hay, needle string) bool {
|
|
return len(hay) >= len(needle) && (indexOf(hay, needle) >= 0)
|
|
}
|
|
|
|
func indexOf(hay, needle string) int {
|
|
for i := 0; i+len(needle) <= len(hay); i++ {
|
|
if hay[i:i+len(needle)] == needle {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|