386 lines
15 KiB
Go
386 lines
15 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/equality"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
bindv1alpha1 "git.unkin.net/unkin/bind-operator/api/v1alpha1"
|
|
"git.unkin.net/unkin/bind-operator/internal/bind"
|
|
)
|
|
|
|
func testScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
if err := clientgoscheme.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := bindv1alpha1.AddToScheme(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// fakeExec mirrors bind.Executor: stdout is returned, stderr only inside the
|
|
// error. It serves `cat` from mounted (per pod) and counts reconfigs.
|
|
type fakeExec struct {
|
|
mounted map[string]string
|
|
catErr map[string]error
|
|
reconfigErr error
|
|
checkconfStderr string
|
|
reconfigs map[string]int
|
|
}
|
|
|
|
func (f *fakeExec) Exec(_ context.Context, _, pod string, cmd []string, _ string) (string, error) {
|
|
if len(cmd) == 2 && cmd[0] == bind.CheckconfBin {
|
|
return "", fmt.Errorf("exec %v: command terminated with exit code 1 (stderr: %s)", cmd, f.checkconfStderr)
|
|
}
|
|
if len(cmd) != 3 || cmd[0] != "cat" || cmd[1] != bind.NamedConfRun || cmd[2] != bind.KeysConfPath {
|
|
return "", errors.New("unexpected command")
|
|
}
|
|
if err := f.catErr[pod]; err != nil {
|
|
return "", err
|
|
}
|
|
return f.mounted[pod], nil
|
|
}
|
|
|
|
func (f *fakeExec) Reconfig(_ context.Context, _, pod string) error {
|
|
f.reconfigs[pod]++
|
|
return f.reconfigErr
|
|
}
|
|
|
|
func readyPod(cluster, name string) *corev1.Pod {
|
|
return &corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", Labels: commonLabels(cluster)},
|
|
Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
|
|
}
|
|
}
|
|
|
|
func TestPodTemplateIgnoresConfigButTracksPodSpec(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
|
|
Spec: bindv1alpha1.BindClusterSpec{Replicas: 2},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build()
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme}
|
|
ctx := context.Background()
|
|
|
|
template := func() corev1.PodTemplateSpec {
|
|
t.Helper()
|
|
sts, err := r.reconcileStatefulSet(ctx, cluster)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sts.Spec.Template
|
|
}
|
|
setConfig := func(conf, keys string) {
|
|
t.Helper()
|
|
if err := r.upsertConfigMap(ctx, cluster, configMapName("c"), map[string]string{"named.conf.secondary": conf}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.upsertSecret(ctx, cluster, keysSecretName("c"), map[string][]byte{"keys.conf": []byte(keys)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
setConfig("options { recursion yes; };", "")
|
|
base := template()
|
|
setConfig(`options { recursion no; }; zone "x" { type forward; forwarders { 1.1.1.1; }; };`, "key k {};")
|
|
if got := template(); !equality.Semantic.DeepEqual(base, got) {
|
|
t.Fatal("pod template changed on a rendered-config change; that restarts every pod")
|
|
}
|
|
|
|
cluster.Spec.Image = "internetsystemsconsortium/bind9:9.21"
|
|
withImage := template()
|
|
if equality.Semantic.DeepEqual(base, withImage) {
|
|
t.Fatal("pod template must change when the image changes")
|
|
}
|
|
|
|
cluster.Spec.Resources = corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}}
|
|
if equality.Semantic.DeepEqual(withImage, template()) {
|
|
t.Fatal("pod template must change when resources change")
|
|
}
|
|
}
|
|
|
|
func TestReconfigPodsWaitsForMountThenReconfigsOnce(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), readyPod("c", "c-1")).Build()
|
|
ex := &fakeExec{
|
|
mounted: map[string]string{"c-0": "old-primary", "c-1": "old-secondary"},
|
|
reconfigs: map[string]int{},
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
ctx := context.Background()
|
|
const primary, secondary, keys = "new-primary;", "new-secondary;", "key k {};"
|
|
|
|
if d := r.reconfigPods(ctx, cluster, primary, secondary, keys); d != requeueShort {
|
|
t.Fatalf("stale mounts should requeue short, got %v", d)
|
|
}
|
|
if len(ex.reconfigs) != 0 {
|
|
t.Fatalf("reconfig ran before the mount caught up: %v", ex.reconfigs)
|
|
}
|
|
if c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied"); c == nil || c.Reason != "Propagating" {
|
|
t.Fatalf("want Propagating condition, got %+v", c)
|
|
}
|
|
|
|
// Only the primary's mount has propagated.
|
|
ex.mounted["c-0"] = primary + keys
|
|
if d := r.reconfigPods(ctx, cluster, primary, secondary, keys); d != requeueShort {
|
|
t.Fatalf("secondary still stale should requeue, got %v", d)
|
|
}
|
|
if ex.reconfigs["c-0"] != 1 || ex.reconfigs["c-1"] != 0 {
|
|
t.Fatalf("want only c-0 reconfigured, got %v", ex.reconfigs)
|
|
}
|
|
|
|
ex.mounted["c-1"] = secondary + keys
|
|
for range 3 {
|
|
if d := r.reconfigPods(ctx, cluster, primary, secondary, keys); d != 0 {
|
|
t.Fatalf("converged cluster should not requeue, got %v", d)
|
|
}
|
|
}
|
|
if ex.reconfigs["c-0"] != 1 || ex.reconfigs["c-1"] != 1 {
|
|
t.Fatalf("each pod must reconfig exactly once, got %v", ex.reconfigs)
|
|
}
|
|
if c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied"); c == nil || c.Status != metav1.ConditionTrue {
|
|
t.Fatalf("want ConfigApplied=True, got %+v", c)
|
|
}
|
|
|
|
// A pod that no longer exists drops out of the applied map.
|
|
if err := cl.Delete(ctx, readyPod("c", "c-1")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.reconfigPods(ctx, cluster, primary, secondary, keys)
|
|
if _, ok := cluster.Status.AppliedConfig["c-1"]; ok {
|
|
t.Fatal("applied map kept a deleted pod")
|
|
}
|
|
}
|
|
|
|
func TestReconfigPodsSurfacesFailure(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0")).Build()
|
|
ex := &fakeExec{
|
|
mounted: map[string]string{"c-0": "bad;"},
|
|
reconfigErr: errors.New("rndc: 'reconfig' failed: unexpected token"),
|
|
reconfigs: map[string]int{},
|
|
checkconfStderr: "/run/named/named.conf:6: expected IP address near 'not-an-ip'",
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
if d := r.reconfigPods(context.Background(), cluster, "bad;", "", ""); d != requeueLong {
|
|
t.Fatalf("failure should requeue long, got %v", d)
|
|
}
|
|
c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied")
|
|
if c == nil || c.Status != metav1.ConditionFalse || c.Reason != "ReconfigFailed" {
|
|
t.Fatalf("want ReconfigFailed condition, got %+v", c)
|
|
}
|
|
if !strings.Contains(c.Message, "named.conf:6: expected IP address") {
|
|
t.Fatalf("condition should carry the checkconf detail, got %q", c.Message)
|
|
}
|
|
if _, ok := cluster.Status.AppliedConfig["c-0"]; ok {
|
|
t.Fatal("failed reconfig must not be recorded as applied")
|
|
}
|
|
}
|
|
|
|
func notReadyPod(cluster, name string) *corev1.Pod {
|
|
p := readyPod(cluster, name)
|
|
p.Status.Conditions[0].Status = corev1.ConditionFalse
|
|
return p
|
|
}
|
|
|
|
func TestReconfigPodsSurfacesExecErrorAndBacksOff(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0")).Build()
|
|
ex := &fakeExec{
|
|
catErr: map[string]error{"c-0": errors.New(`pods "c-0" is forbidden: cannot create resource "pods/exec"`)},
|
|
reconfigs: map[string]int{},
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
if d := r.reconfigPods(context.Background(), cluster, "p;", "s;", ""); d != requeueLong {
|
|
t.Fatalf("exec error should back off with requeueLong, got %v", d)
|
|
}
|
|
c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied")
|
|
if c == nil || c.Status != metav1.ConditionFalse || c.Reason != "ExecFailed" || !strings.Contains(c.Message, "forbidden") {
|
|
t.Fatalf("want ExecFailed condition carrying the cause, got %+v", c)
|
|
}
|
|
if ex.reconfigs["c-0"] != 0 {
|
|
t.Fatal("reconfig ran after a failed mount check")
|
|
}
|
|
}
|
|
|
|
func TestReconfigPodsNotReadyStalePodBlocksApplied(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), notReadyPod("c", "c-1")).Build()
|
|
const primary, secondary = "p;", "s;"
|
|
ex := &fakeExec{
|
|
mounted: map[string]string{"c-0": primary},
|
|
catErr: map[string]error{"c-1": errors.New("must not exec into a not-ready pod")},
|
|
reconfigs: map[string]int{},
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
if d := r.reconfigPods(context.Background(), cluster, primary, secondary, ""); d != requeueShort {
|
|
t.Fatalf("not-ready stale pod should requeue short, got %v", d)
|
|
}
|
|
c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied")
|
|
if c == nil || c.Status != metav1.ConditionFalse || c.Reason != "Propagating" || !strings.Contains(c.Message, "c-1") {
|
|
t.Fatalf("want Propagating naming c-1, got %+v", c)
|
|
}
|
|
if ex.reconfigs["c-0"] != 1 || ex.reconfigs["c-1"] != 0 {
|
|
t.Fatalf("want only the ready pod reconfigured, got %v", ex.reconfigs)
|
|
}
|
|
}
|
|
|
|
func TestReconfigPodsNewPodReconfigsOnceThenRecorded(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
const primary, secondary = "p;", "s;"
|
|
cluster := &bindv1alpha1.BindCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
|
|
Status: bindv1alpha1.BindClusterStatus{AppliedConfig: map[string]string{"c-0": sha256Hex(primary)}},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), readyPod("c", "c-1")).Build()
|
|
ex := &fakeExec{mounted: map[string]string{"c-0": primary, "c-1": secondary}, reconfigs: map[string]int{}}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
for range 3 {
|
|
if d := r.reconfigPods(context.Background(), cluster, primary, secondary, ""); d != 0 {
|
|
t.Fatalf("converged cluster should not requeue, got %v", d)
|
|
}
|
|
}
|
|
if ex.reconfigs["c-0"] != 0 || ex.reconfigs["c-1"] != 1 {
|
|
t.Fatalf("want exactly one reconfig for the new pod only, got %v", ex.reconfigs)
|
|
}
|
|
if cluster.Status.AppliedConfig["c-1"] != sha256Hex(secondary) {
|
|
t.Fatalf("new pod not recorded: %v", cluster.Status.AppliedConfig)
|
|
}
|
|
}
|
|
|
|
func TestReconcilePersistsAppliedConfig(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
|
|
Spec: bindv1alpha1.BindClusterSpec{Replicas: 1},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).
|
|
WithObjects(cluster, readyPod("c", "c-0")).
|
|
WithStatusSubresource(&bindv1alpha1.BindCluster{}).
|
|
Build()
|
|
ex := &fakeExec{mounted: map[string]string{}, reconfigs: map[string]int{}}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
ctx := context.Background()
|
|
req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "c"}}
|
|
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cm corev1.ConfigMap
|
|
if err := cl.Get(ctx, types.NamespacedName{Namespace: "ns", Name: configMapName("c")}, &cm); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var keys corev1.Secret
|
|
if err := cl.Get(ctx, types.NamespacedName{Namespace: "ns", Name: keysSecretName("c")}, &keys); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mounted := cm.Data["named.conf.primary"] + string(keys.Data["keys.conf"])
|
|
ex.mounted["c-0"] = mounted
|
|
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got bindv1alpha1.BindCluster
|
|
if err := cl.Get(ctx, req.NamespacedName, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status.AppliedConfig["c-0"] != sha256Hex(mounted) {
|
|
t.Fatalf("appliedConfig not persisted: %v", got.Status.AppliedConfig)
|
|
}
|
|
if _, err := r.Reconcile(ctx, req); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ex.reconfigs["c-0"] != 1 {
|
|
t.Fatalf("persisted appliedConfig should prevent a second reconfig, got %d", ex.reconfigs["c-0"])
|
|
}
|
|
}
|
|
|
|
func TestReconfigPodsFailureBacksOffWhileOthersWait(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), notReadyPod("c", "c-1")).Build()
|
|
ex := &fakeExec{
|
|
mounted: map[string]string{"c-0": "p;"},
|
|
reconfigErr: errors.New("rndc: 'reconfig' failed"),
|
|
reconfigs: map[string]int{},
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
if d := r.reconfigPods(context.Background(), cluster, "p;", "s;", ""); d != requeueLong {
|
|
t.Fatalf("a failed pod must back off with requeueLong even while another waits, got %v", d)
|
|
}
|
|
if c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied"); c == nil || c.Reason != "ReconfigFailed" {
|
|
t.Fatalf("want ReconfigFailed, got %+v", c)
|
|
}
|
|
}
|
|
|
|
func TestReconcileFailureBacksOffWhileStatefulSetNotReady(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
|
|
Spec: bindv1alpha1.BindClusterSpec{Replicas: 2},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).
|
|
WithObjects(cluster, readyPod("c", "c-0")).
|
|
WithStatusSubresource(&bindv1alpha1.BindCluster{}).
|
|
Build()
|
|
ex := &fakeExec{
|
|
catErr: map[string]error{"c-0": errors.New("forbidden")},
|
|
reconfigs: map[string]int{},
|
|
}
|
|
r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
|
|
|
|
res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "c"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.RequeueAfter != requeueLong {
|
|
t.Fatalf("a failed pod must back off with requeueLong while the StatefulSet is not Ready, got %v", res.RequeueAfter)
|
|
}
|
|
}
|
|
|
|
func TestPodTemplateEntrypointHashTracksScript(t *testing.T) {
|
|
scheme := testScheme(t)
|
|
cluster := &bindv1alpha1.BindCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
|
|
Spec: bindv1alpha1.BindClusterSpec{Replicas: 1},
|
|
}
|
|
r := &BindClusterReconciler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build(), Scheme: scheme}
|
|
sts, err := r.reconcileStatefulSet(context.Background(), cluster)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := sts.Spec.Template.Annotations[entrypointHashAnnotation]
|
|
if got != sha256Hex(entrypointScript()) {
|
|
t.Fatalf("entrypoint hash annotation %q does not match the script", got)
|
|
}
|
|
if got == sha256Hex(entrypointScript()+"\n# changed") {
|
|
t.Fatal("entrypoint hash annotation must change when the script changes")
|
|
}
|
|
}
|