surface exec and checkconf errors on ConfigApplied, count not-ready stale pods as waiting

internal/controller/bindcluster_controller.go

@@ -472,7 +472,7 @@ func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha
 		return requeueShort
 	}
 	applied := map[string]string{}
-	var waiting, failed []string
+	var waiting, failed, execErrs []string
 	for i := range pods.Items {
 		pod := &pods.Items[i]
 		conf := secondaryConf
@@ -483,18 +483,30 @@ func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha
 		if have, ok := c.Status.AppliedConfig[pod.Name]; ok {
 			applied[pod.Name] = have
 		}
-		if applied[pod.Name] == want || !podReady(pod) {
+		if applied[pod.Name] == want {
+			continue
+		}
+		if !podReady(pod) {
+			waiting = append(waiting, pod.Name)
 			continue
 		}
 		mounted, err := r.Exec.Exec(ctx, c.Namespace, pod.Name, []string{"cat", bind.NamedConfRun, bind.KeysConfPath}, "")
-		if err != nil || sha256Hex(mounted) != want {
+		if err != nil {
+			execErrs = append(execErrs, fmt.Sprintf("%s: %v", pod.Name, err))
+			continue
+		}
+		if sha256Hex(mounted) != want {
 			waiting = append(waiting, pod.Name)
 			continue
 		}
 		if err := r.Exec.Reconfig(ctx, c.Namespace, pod.Name); err != nil {
-			// rndc only reports "failed: <reason>"; checkconf names the file and line.
-			detail, _ := r.Exec.Exec(ctx, c.Namespace, pod.Name, []string{bind.CheckconfBin, bind.NamedConfRun}, "")
-			failed = append(failed, strings.TrimSpace(fmt.Sprintf("%s: %v %s", pod.Name, err, detail)))
+			// rndc only reports "failed: <reason>"; checkconf names the file and line (on stderr, so in the error).
+			out, cerr := r.Exec.Exec(ctx, c.Namespace, pod.Name, []string{bind.CheckconfBin, bind.NamedConfRun}, "")
+			msg := fmt.Sprintf("%s: %v", pod.Name, err)
+			if cerr != nil {
+				msg += "; " + strings.TrimSpace(out+" "+cerr.Error())
+			}
+			failed = append(failed, msg)
 			continue
 		}
 		applied[pod.Name] = want
@@ -505,12 +517,16 @@ func (r *BindClusterReconciler) reconfigPods(ctx context.Context, c *bindv1alpha
 	var requeue time.Duration
 	switch {
 	case len(failed) > 0:
-		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "ReconfigFailed", strings.Join(failed, "; ")
-		requeue = requeueLong
-		log.FromContext(ctx).Info("rndc reconfig failed", "cluster", c.Name, "errors", cond.Message)
+		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "ReconfigFailed", strings.Join(append(failed, execErrs...), "; ")
+	case len(execErrs) > 0:
+		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "ExecFailed", strings.Join(execErrs, "; ")
 	case len(waiting) > 0:
 		cond.Status, cond.Reason, cond.Message = metav1.ConditionFalse, "Propagating", "awaiting config on "+strings.Join(waiting, ", ")
 	}
+	if len(failed)+len(execErrs) > 0 {
+		requeue = requeueLong
+		log.FromContext(ctx).Info("config apply failed", "cluster", c.Name, "reason", cond.Reason, "errors", cond.Message)
+	}
 	if len(waiting) > 0 {
 		requeue = requeueShort
 	}