apply config changes via rndc reconfig instead of restarting pods

- symlink named.conf to the projected file so kubelet updates reach named
- drop the config hash from the pod template; hash only the entrypoint
- reconfig each pod once its mounted config matches, tracked in status
- surface reconfig failures with checkconf detail on ConfigApplied

internal/controller/bindcluster_controller.go

@@ -1,17 +1,18 @@
 package controller
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -32,7 +33,13 @@ import (
 type BindClusterReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
-	Exec   *bind.Executor
+	Exec   podExecutor
+}
+
+// podExecutor is the subset of *bind.Executor the cluster reconciler uses.
+type podExecutor interface {
+	Exec(ctx context.Context, namespace, pod string, command []string, stdin string) (string, error)
+	Reconfig(ctx context.Context, namespace, pod string) error
 }
 
 // +kubebuilder:rbac:groups=bind.unkin.net,resources=bindclusters,verbs=get;list;watch;create;update;patch;delete
@@ -54,10 +61,12 @@ func (r *BindClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if err := r.reconcileRNDCSecret(ctx, &cluster); err != nil {
 		return ctrl.Result{}, fmt.Errorf("rndc secret: %w", err)
 	}
-	if err := r.reconcileKeysSecret(ctx, &cluster); err != nil {
+	keysConf, err := r.reconcileKeysSecret(ctx, &cluster)
+	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("keys secret: %w", err)
 	}
-	if err := r.reconcileConfigMap(ctx, &cluster); err != nil {
+	primaryConf, secondaryConf, err := r.reconcileConfigMap(ctx, &cluster)
+	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("configmap: %w", err)
 	}
 	if err := r.reconcileServices(ctx, &cluster); err != nil {
@@ -68,9 +77,7 @@ func (r *BindClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, fmt.Errorf("statefulset: %w", err)
 	}
 
-	// Best-effort: reload configuration on ready pods so ConfigMap changes take
-	// effect without a rollout.
-	r.reloadReadyPods(ctx, &cluster)
+	requeue := r.reconfigPods(ctx, &cluster, primaryConf, secondaryConf, keysConf)
 
 	// Status.
 	cluster.Status.ObservedGeneration = cluster.Generation
@@ -91,7 +98,10 @@ func (r *BindClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	if !ready {
-		return ctrl.Result{RequeueAfter: requeueShort}, nil
+		requeue = requeueShort
+	}
+	if requeue > 0 {
+		return ctrl.Result{RequeueAfter: requeue}, nil
 	}
 	logger.V(1).Info("cluster reconciled", "cluster", cluster.Name, "ready", sts.Status.ReadyReplicas)
 	return ctrl.Result{}, nil
@@ -126,10 +136,10 @@ func (r *BindClusterReconciler) reconcileRNDCSecret(ctx context.Context, c *bind
 	return r.Create(ctx, s)
 }
 
-func (r *BindClusterReconciler) reconcileKeysSecret(ctx context.Context, c *bindv1alpha1.BindCluster) error {
+func (r *BindClusterReconciler) reconcileKeysSecret(ctx context.Context, c *bindv1alpha1.BindCluster) (string, error) {
 	var keys bindv1alpha1.BindTSIGKeyList
 	if err := r.List(ctx, &keys, client.InNamespace(c.Namespace)); err != nil {
-		return err
+		return "", err
 	}
 	// Include keys scoped to this cluster (spec.clusterRef == name) and shared
 	// keys (empty clusterRef). This keeps keys from leaking across clusters that
@@ -167,10 +177,10 @@ func (r *BindClusterReconciler) reconcileKeysSecret(ctx context.Context, c *bind
 		b.WriteString(bind.KeyClause(keyName, alg, string(secret.Data["secret"])))
 	}
 
-	return r.upsertSecret(ctx, c, keysSecretName(c.Name), map[string][]byte{"keys.conf": []byte(b.String())})
+	return b.String(), r.upsertSecret(ctx, c, keysSecretName(c.Name), map[string][]byte{"keys.conf": []byte(b.String())})
 }
 
-func (r *BindClusterReconciler) reconcileConfigMap(ctx context.Context, c *bindv1alpha1.BindCluster) error {
+func (r *BindClusterReconciler) reconcileConfigMap(ctx context.Context, c *bindv1alpha1.BindCluster) (primaryConf, secondaryConf string, err error) {
 	// BIND primaries/default-primaries need an IP address, not a DNS name. Use
 	// the stable primary Service ClusterIP so secondaries keep transferring
 	// across primary pod restarts (falls back to the pod IP when no primary
@@ -241,13 +251,13 @@ func (r *BindClusterReconciler) reconcileConfigMap(ctx context.Context, c *bindv
 		in.NotifyKeyName = tsigKeyName(ctx, r.Client, c.Namespace, in.Catalog.Spec.TransferKeyRef)
 	}
 
-	primaryConf, secondaryConf := bind.RenderNamedConf(in)
+	primaryConf, secondaryConf = bind.RenderNamedConf(in)
 	data := map[string]string{
 		"named.conf.primary":   primaryConf,
 		"named.conf.secondary": secondaryConf,
 		"entrypoint.sh":        entrypointScript(),
 	}
-	return r.upsertConfigMap(ctx, c, configMapName(c.Name), data)
+	return primaryConf, secondaryConf, r.upsertConfigMap(ctx, c, configMapName(c.Name), data)
 }
 
 func (r *BindClusterReconciler) reconcileServices(ctx context.Context, c *bindv1alpha1.BindCluster) error {
@@ -364,12 +374,8 @@ func (r *BindClusterReconciler) reconcileStatefulSet(ctx context.Context, c *bin
 		}}},
 	}
 
-	// Pods copy config from the projected volume into an emptyDir once at
-	// startup; a ConfigMap/keys change never reaches a running pod (rndc
-	// reconfig re-reads the stale startup copy). Stamp a hash of the projected
-	// config onto the pod template so a config change rolls the StatefulSet,
-	// which is the only way the change takes effect. The operator owns the
-	// template, so this restart is operator-driven and not reverted.
+	// Rendered config is applied in place by reconfigPods; only an entrypoint
+	// change needs a restart, so only its hash goes on the template.
 	sts := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{Name: c.Name, Namespace: c.Namespace, Labels: labels},
 		Spec: appsv1.StatefulSetSpec{
@@ -377,7 +383,7 @@ func (r *BindClusterReconciler) reconcileStatefulSet(ctx context.Context, c *bin
 			Replicas:    &replicas,
 			Selector:    &metav1.LabelSelector{MatchLabels: labels},
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: map[string]string{configHashAnnotation: r.configHash(ctx, c)}},
+				ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: map[string]string{entrypointHashAnnotation: sha256Hex(entrypointScript())}},
 				Spec: corev1.PodSpec{
 					NodeSelector: c.Spec.NodeSelector,
 					Tolerations:  c.Spec.Tolerations,
@@ -445,63 +451,71 @@ func (r *BindClusterReconciler) reconcileStatefulSet(ctx context.Context, c *bin
 	return &existing, nil
 }
 
-// configHashAnnotation carries a hash of the projected config on the pod
-// template; changing it triggers a rolling restart so pods pick up new config.
-const configHashAnnotation = "bind.unkin.net/config-hash"
+// entrypointHashAnnotation rolls the pods when the entrypoint script changes.
+const entrypointHashAnnotation = "bind.unkin.net/entrypoint-hash"
 
-// configHash returns a deterministic hash of the config projected into the pods
-// (the rendered ConfigMap and the keys.conf Secret). It is read after those are
-// reconciled, so it reflects the current desired config. A stable hash means no
-// spurious restarts; any config or TSIG-key change flips it and rolls the pods.
-func (r *BindClusterReconciler) configHash(ctx context.Context, c *bindv1alpha1.BindCluster) string {
-	var buf bytes.Buffer
-	var cm corev1.ConfigMap
-	if err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: configMapName(c.Name)}, &cm); err == nil {
-		for _, k := range sortedKeys(cm.Data) {
-			fmt.Fprintf(&buf, "%s\x00%s\x00", k, cm.Data[k])
-		}
-	}
-	var keys corev1.Secret
-	if err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: keysSecretName(c.Name)}, &keys); err == nil {
-		data := make(map[string]string, len(keys.Data))
-		for k, v := range keys.Data {
-			data[k] = string(v)
-		}
-		for _, k := range sortedKeys(data) {
-			fmt.Fprintf(&buf, "%s\x00%s\x00", k, data[k])
-		}
-	}
-	sum := sha256.Sum256(buf.Bytes())
+func sha256Hex(s string) string {
+	sum := sha256.Sum256([]byte(s))
 	return hex.EncodeToString(sum[:])
 }
 
-func sortedKeys(m map[string]string) []string {
-	keys := make([]string, 0, len(m))
-	for k := range m {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	return keys
-}
-
-func (r *BindClusterReconciler) reloadReadyPods(ctx context.Context, c *bindv1alpha1.BindCluster) {
+// reconfigPods runs rndc reconfig on each ready pod whose applied config is not
+// the desired one, once kubelet has propagated it to the pod's mount (eventually
+// consistent, up to ~90s). It records applied hashes in status and sets the
+// ConfigApplied condition, returning a requeue delay while any pod is pending.
+func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha1.BindCluster, primaryConf, secondaryConf, keysConf string) time.Duration {
 	if r.Exec == nil {
-		return
+		return 0
 	}
-	logger := log.FromContext(ctx)
 	var pods corev1.PodList
 	if err := r.List(ctx, &pods, client.InNamespace(c.Namespace), client.MatchingLabels(commonLabels(c.Name))); err != nil {
-		return
+		return requeueShort
 	}
+	applied := map[string]string{}
+	var waiting, failed []string
 	for i := range pods.Items {
 		pod := &pods.Items[i]
-		if !podReady(pod) {
+		conf := secondaryConf
+		if pod.Name == primaryPodName(c.Name) {
+			conf = primaryConf
+		}
+		want := sha256Hex(conf + keysConf)
+		if have, ok := c.Status.AppliedConfig[pod.Name]; ok {
+			applied[pod.Name] = have
+		}
+		if applied[pod.Name] == want || !podReady(pod) {
+			continue
+		}
+		mounted, err := r.Exec.Exec(ctx, c.Namespace, pod.Name, []string{"cat", bind.NamedConfRun, bind.KeysConfPath}, "")
+		if err != nil || sha256Hex(mounted) != want {
+			waiting = append(waiting, pod.Name)
 			continue
 		}
 		if err := r.Exec.Reconfig(ctx, c.Namespace, pod.Name); err != nil {
-			logger.V(1).Info("rndc reconfig failed", "pod", pod.Name, "err", err.Error())
+			// rndc only reports "failed: <reason>"; checkconf names the file and line.
+			detail, _ := r.Exec.Exec(ctx, c.Namespace, pod.Name, []string{bind.CheckconfBin, bind.NamedConfRun}, "")
+			failed = append(failed, strings.TrimSpace(fmt.Sprintf("%s: %v %s", pod.Name, err, detail)))
+			continue
 		}
+		applied[pod.Name] = want
 	}
+	c.Status.AppliedConfig = applied
+
+	cond := metav1.Condition{Type: "ConfigApplied", Status: metav1.ConditionTrue, Reason: "Applied", ObservedGeneration: c.Generation}
+	var requeue time.Duration
+	switch {
+	case len(failed) > 0:
+		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "ReconfigFailed", strings.Join(failed, "; ")
+		requeue = requeueLong
+		log.FromContext(ctx).Info("rndc reconfig failed", "cluster", c.Name, "errors", cond.Message)
+	case len(waiting) > 0:
+		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "Propagating", "awaiting config on "+strings.Join(waiting, ", ")
+	}
+	if len(waiting) > 0 {
+		requeue = requeueShort
+	}
+	meta.SetStatusCondition(&c.Status.Conditions, cond)
+	return requeue
 }
 
 func (r *BindClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {