keep the 2m backoff on config apply failures while pods are waiting or the StatefulSet is not ready

internal/controller/bindcluster_controller.go

@@ -97,7 +97,7 @@ func (r *BindClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
-	if !ready {
+	if !ready && requeue == 0 {
 		requeue = requeueShort
 	}
 	if requeue > 0 {
@@ -462,7 +462,8 @@ func sha256Hex(s string) string {
 // reconfigPods runs rndc reconfig on each ready pod whose applied config is not
 // the desired one, once kubelet has propagated it to the pod's mount (eventually
 // consistent, up to ~90s). It records applied hashes in status and sets the
-// ConfigApplied condition, returning a requeue delay while any pod is pending.
+// ConfigApplied condition, returning a requeue delay while any pod is pending
+// (requeueLong if any pod failed, so waiting pods cannot hot-loop the failure).
 func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha1.BindCluster, primaryConf, secondaryConf, keysConf string) time.Duration {
 	if r.Exec == nil {
 		return 0
@@ -526,8 +527,7 @@ func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha
 	if len(failed)+len(execErrs) > 0 {
 		requeue = requeueLong
 		log.FromContext(ctx).Info("config apply failed", "cluster", c.Name, "reason", cond.Reason, "errors", cond.Message)
-	}
-	if len(waiting) > 0 {
+	} else if len(waiting) > 0 {
 		requeue = requeueShort
 	}
 	meta.SetStatusCondition(&c.Status.Conditions, cond)

internal/controller/reconfig_test.go

@@ -319,3 +319,67 @@ func TestReconcilePersistsAppliedConfig(t *testing.T) {
 		t.Fatalf("persisted appliedConfig should prevent a second reconfig, got %d", ex.reconfigs["c-0"])
 	}
 }
+
+func TestReconfigPodsFailureBacksOffWhileOthersWait(t *testing.T) {
+	scheme := testScheme(t)
+	cluster := &bindv1alpha1.BindCluster{ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"}}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(readyPod("c", "c-0"), notReadyPod("c", "c-1")).Build()
+	ex := &fakeExec{
+		mounted:     map[string]string{"c-0": "p;"},
+		reconfigErr: errors.New("rndc: 'reconfig' failed"),
+		reconfigs:   map[string]int{},
+	}
+	r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
+
+	if d := r.reconfigPods(context.Background(), cluster, "p;", "s;", ""); d != requeueLong {
+		t.Fatalf("a failed pod must back off with requeueLong even while another waits, got %v", d)
+	}
+	if c := meta.FindStatusCondition(cluster.Status.Conditions, "ConfigApplied"); c == nil || c.Reason != "ReconfigFailed" {
+		t.Fatalf("want ReconfigFailed, got %+v", c)
+	}
+}
+
+func TestReconcileFailureBacksOffWhileStatefulSetNotReady(t *testing.T) {
+	scheme := testScheme(t)
+	cluster := &bindv1alpha1.BindCluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
+		Spec:       bindv1alpha1.BindClusterSpec{Replicas: 2},
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).
+		WithObjects(cluster, readyPod("c", "c-0")).
+		WithStatusSubresource(&bindv1alpha1.BindCluster{}).
+		Build()
+	ex := &fakeExec{
+		catErr:    map[string]error{"c-0": errors.New("forbidden")},
+		reconfigs: map[string]int{},
+	}
+	r := &BindClusterReconciler{Client: cl, Scheme: scheme, Exec: ex}
+
+	res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "c"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.RequeueAfter != requeueLong {
+		t.Fatalf("a failed pod must back off with requeueLong while the StatefulSet is not Ready, got %v", res.RequeueAfter)
+	}
+}
+
+func TestPodTemplateEntrypointHashTracksScript(t *testing.T) {
+	scheme := testScheme(t)
+	cluster := &bindv1alpha1.BindCluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "c", Namespace: "ns"},
+		Spec:       bindv1alpha1.BindClusterSpec{Replicas: 1},
+	}
+	r := &BindClusterReconciler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build(), Scheme: scheme}
+	sts, err := r.reconcileStatefulSet(context.Background(), cluster)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := sts.Spec.Template.Annotations[entrypointHashAnnotation]
+	if got != sha256Hex(entrypointScript()) {
+		t.Fatalf("entrypoint hash annotation %q does not match the script", got)
+	}
+	if got == sha256Hex(entrypointScript()+"\n# changed") {
+		t.Fatal("entrypoint hash annotation must change when the script changes")
+	}
+}