diff --git a/internal/controller/bindcluster_controller.go b/internal/controller/bindcluster_controller.go index a9b6204..ae71196 100644 --- a/internal/controller/bindcluster_controller.go +++ b/internal/controller/bindcluster_controller.go @@ -97,7 +97,7 @@ func (r *BindClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - if !ready { + if !ready && requeue == 0 { requeue = requeueShort } if requeue > 0 { @@ -462,7 +462,8 @@ func sha256Hex(s string) string { // reconfigPods runs rndc reconfig on each ready pod whose applied config is not // the desired one, once kubelet has propagated it to the pod's mount (eventually // consistent, up to ~90s). It records applied hashes in status and sets the -// ConfigApplied condition, returning a requeue delay while any pod is pending. +// ConfigApplied condition, returning a requeue delay while any pod is pending +// (requeueLong if any pod failed, so waiting pods cannot hot-loop the failure). func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha1.BindCluster, primaryConf, secondaryConf, keysConf string) time.Duration { if r.Exec == nil { return 0 @@ -526,8 +527,7 @@ func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha if len(failed)+len(execErrs) > 0 { requeue = requeueLong log.FromContext(ctx).Info("config apply failed", "cluster", c.Name, "reason", cond.Reason, "errors", cond.Message) - } - if len(waiting) > 0 { + } else if len(waiting) > 0 { requeue = requeueShort } meta.SetStatusCondition(&c.Status.Conditions, cond) diff --git a/internal/controller/reconfig_test.go b/internal/controller/reconfig_test.go index a486831..d491fed 100644 --- a/internal/controller/reconfig_test.go +++ b/internal/controller/reconfig_test.go @@ -319,3 +319,67 @@ func TestReconcilePersistsAppliedConfig(t *testing.T) { t.Fatalf("persisted appliedConfig should prevent a second reconfig, got %d", ex.reconfigs["c-0"]) } } + +func TestReconfigPodsFailureBacksOffWhileOthersWait(t *testing.T) { + scheme := testScheme(t) + cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}} + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), notReadyPod("c", "c-1")).Build() + ex := &fakeExec{ + mounted: map[string]string{"c-0": "p;"}, + reconfigErr: errors.New("rndc: 'reconfig' failed"), + reconfigs: map[string]int{}, + } + r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex} + + if d := r.reconfigPods(context.Background(), cluster, "p;", "s;", ""); d != requeueLong { + t.Fatalf("a failed pod must back off with requeueLong even while another waits, got %v", d) + } + if c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied"); c == nil || c.Reason != "ReconfigFailed" { + t.Fatalf("want ReconfigFailed, got %+v", c) + } +} + +func TestReconcileFailureBacksOffWhileStatefulSetNotReady(t *testing.T) { + scheme := testScheme(t) + cluster := &bindv1alpha1.BindCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}, + Spec: bindv1alpha1.BindClusterSpec{Replicas: 2}, + } + cl := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, readyPod("c", "c-0")). + WithStatusSubresource(&bindv1alpha1.BindCluster{}). + Build() + ex := &fakeExec{ + catErr: map[string]error{"c-0": errors.New("forbidden")}, + reconfigs: map[string]int{}, + } + r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex} + + res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "c"}}) + if err != nil { + t.Fatal(err) + } + if res.RequeueAfter != requeueLong { + t.Fatalf("a failed pod must back off with requeueLong while the StatefulSet is not Ready, got %v", res.RequeueAfter) + } +} + +func TestPodTemplateEntrypointHashTracksScript(t *testing.T) { + scheme := testScheme(t) + cluster := &bindv1alpha1.BindCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}, + Spec: bindv1alpha1.BindClusterSpec{Replicas: 1}, + } + r := &BindClusterReconciler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build(), Scheme: scheme} + sts, err := r.reconcileStatefulSet(context.Background(), cluster) + if err != nil { + t.Fatal(err) + } + got := sts.Spec.Template.Annotations[entrypointHashAnnotation] + if got != sha256Hex(entrypointScript()) { + t.Fatalf("entrypoint hash annotation %q does not match the script", got) + } + if got == sha256Hex(entrypointScript()+"\n# changed") { + t.Fatal("entrypoint hash annotation must change when the script changes") + } +}