Scaffold kea-operator: CRDs, controllers, config rendering, REST API, CI
ci/woodpecker/pr/build Pipeline failed
ci/woodpecker/pr/pre-commit Pipeline failed
ci/woodpecker/pr/test Pipeline failed

Replace the ISC dhcpd PXE-boot VM with a Kea DHCP Kubernetes operator, modelled
on bind-operator. The operator renders kea-dhcp4 config from CRs and runs an HA
pair of kea-dhcp4 + kea-ctrl-agent servers behind an anycast Service.

- add KeaCluster/KeaSubnet/KeaClientClass/KeaAPI CRDs (group kea.unkin.net)
- render deterministic kea-dhcp4.conf + kea-ctrl-agent.conf into a ConfigMap and
  roll the StatefulSet via a config-hash annotation; best-effort hot-reload via
  the kea-ctrl-agent REST channel
- run HA hot-standby (memfile leases) with stable per-peer DNS identity from a
  StatefulSet; expose an anycast LoadBalancer Service for PureLB
- represent the full legacy dhcpd config: 198.18.13-17.0/24 pools, pool-less
  198.18.25.0/24, and the Legacy/UEFI-64 PXE arch classes (option 93)
- add the KeaAPI-spawned REST service: Terraform-friendly CRUD over subnet and
  client-class CRs (stable IDs, PUT upsert, 404 drift, bearer-token auth)
- add Makefile (patch/minor/major tag targets), distroless operator/api images,
  an AlmaLinux+EPEL kea workload image, and woodpecker CI with k8s resources +
  serviceAccountName on every step
- unit tests for config rendering, controller reconcile/config-hash, and the API

Claude-Session: https://claude.ai/code/session_01JUoARVdmhxKQHyyyp1pxeT
This commit is contained in:
unkinben
2026-08-02 17:19:53 +10:00
parent 9d471b0bff
commit d3fb5dcd1a
50 changed files with 10379 additions and 1 deletions
+104
View File
@@ -0,0 +1,104 @@
package controller
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func intstrFromInt(i int) intstr.IntOrString { return intstr.FromInt(i) }
const (
requeueShort = 15 * time.Second
requeueLong = 2 * time.Minute
managedByLabel = "app.kubernetes.io/managed-by"
managedByValue = "kea-operator"
clusterLabel = "kea.unkin.net/cluster"
roleLabel = "kea.unkin.net/role"
finalizer = "kea.unkin.net/finalizer"
defaultOperatorImage = "git.unkin.net/unkin/kea-operator:latest"
defaultAPIImage = "git.unkin.net/unkin/kea-api:latest"
)
func headlessName(cluster string) string { return cluster + "-headless" }
func serviceName(cluster string) string { return cluster }
func configMapName(cluster string) string { return cluster + "-config" }
func stsName(cluster string) string { return cluster }
func peerDNS(cluster, ns string, ordinal int) string {
return fmt.Sprintf("http://%s-%d.%s.%s:%d/", cluster, ordinal, headlessName(cluster), ns, 8000)
}
func commonLabels(cluster string) map[string]string {
return map[string]string{
managedByLabel: managedByValue,
clusterLabel: cluster,
}
}
// setReady sets the single "Ready" condition, mirroring bind-operator.
func setReady(conds *[]metav1.Condition, gen int64, ok bool, reason, msg string) {
status := metav1.ConditionFalse
if ok {
status = metav1.ConditionTrue
}
meta.SetStatusCondition(conds, metav1.Condition{
Type: "Ready",
Status: status,
ObservedGeneration: gen,
Reason: reason,
Message: msg,
})
}
// configHash returns a stable hash of the named ConfigMap's data. Pods copy
// config out of a projected volume at startup, so a ConfigMap change alone
// never reaches a running pod; stamping this hash on the pod template is what
// rolls the StatefulSet. The input must contain no pod IPs or the roll loops.
func configHash(ctx context.Context, c client.Client, ns, name string) (string, error) {
var cm corev1.ConfigMap
if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &cm); err != nil {
return "", err
}
keys := make([]string, 0, len(cm.Data))
for k := range cm.Data {
keys = append(keys, k)
}
sort.Strings(keys)
var buf bytes.Buffer
for _, k := range keys {
buf.WriteString(k)
buf.WriteByte(0)
buf.WriteString(cm.Data[k])
buf.WriteByte(0)
}
sum := sha256.Sum256(buf.Bytes())
return hex.EncodeToString(sum[:]), nil
}
func int32ptr(i int32) *int32 { return &i }
func ptr[T any](v T) *T { return &v }
// splitPool splits a "start - end" (any spacing) or single-address pool into
// its address fields.
func splitPool(p string) []string {
if !strings.Contains(p, "-") {
return []string{strings.TrimSpace(p)}
}
parts := strings.SplitN(p, "-", 2)
return []string{strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])}
}
+253
View File
@@ -0,0 +1,253 @@
package controller
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
)
// KeaAPIReconciler spawns the REST API service that CRUDs KeaSubnet and
// KeaClientClass CRs (a Terraform-friendly alternative to argocd-managed CRs).
type KeaAPIReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaapis,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaapis/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=serviceaccounts;secrets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete
func (r *KeaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var api v1alpha1.KeaAPI
if err := r.Get(ctx, req.NamespacedName, &api); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
for _, step := range []func(context.Context, *v1alpha1.KeaAPI) error{
r.reconcileServiceAccount,
r.reconcileRBAC,
r.reconcileTokenSecret,
r.reconcileDeployment,
r.reconcileService,
} {
if err := step(ctx, &api); err != nil {
api.Status.Phase = "Error"
setReady(&api.Status.Conditions, api.Generation, false, "ReconcileError", err.Error())
_ = r.Status().Update(ctx, &api)
return ctrl.Result{}, err
}
}
var dep appsv1.Deployment
_ = r.Get(ctx, types.NamespacedName{Namespace: api.Namespace, Name: api.Name}, &dep)
port := api.Spec.Service.Port
if port == 0 {
port = 8080
}
api.Status.ReadyReplicas = dep.Status.ReadyReplicas
api.Status.Endpoint = fmt.Sprintf("http://%s.%s.svc:%d", api.Name, api.Namespace, port)
api.Status.ObservedGeneration = api.Generation
if dep.Status.ReadyReplicas > 0 {
api.Status.Phase = "Ready"
setReady(&api.Status.Conditions, api.Generation, true, "Ready", "api ready")
} else {
api.Status.Phase = "Progressing"
setReady(&api.Status.Conditions, api.Generation, false, "Progressing", "waiting for api pods")
}
if err := r.Status().Update(ctx, &api); err != nil {
return ctrl.Result{}, err
}
if api.Status.Phase != "Ready" {
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
return ctrl.Result{RequeueAfter: requeueLong}, nil
}
func (r *KeaAPIReconciler) reconcileServiceAccount(ctx context.Context, api *v1alpha1.KeaAPI) error {
sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: api.Name, Namespace: api.Namespace}}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, sa, func() error {
sa.Labels = apiLabels(api.Name)
return ctrl.SetControllerReference(api, sa, r.Scheme)
})
return err
}
func (r *KeaAPIReconciler) reconcileRBAC(ctx context.Context, api *v1alpha1.KeaAPI) error {
role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: api.Name, Namespace: api.Namespace}}
if _, err := ctrl.CreateOrUpdate(ctx, r.Client, role, func() error {
role.Labels = apiLabels(api.Name)
role.Rules = []rbacv1.PolicyRule{
{
APIGroups: []string{"kea.unkin.net"},
Resources: []string{"keasubnets", "keaclientclasses"},
Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"},
},
{
APIGroups: []string{"kea.unkin.net"},
Resources: []string{"keasubnets/status", "keaclientclasses/status"},
Verbs: []string{"get"},
},
}
return ctrl.SetControllerReference(api, role, r.Scheme)
}); err != nil {
return err
}
rb := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: api.Name, Namespace: api.Namespace}}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, rb, func() error {
rb.Labels = apiLabels(api.Name)
rb.RoleRef = rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: api.Name}
rb.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: api.Name, Namespace: api.Namespace}}
return ctrl.SetControllerReference(api, rb, r.Scheme)
})
return err
}
// reconcileTokenSecret creates the bearer-token Secret only when absent, so it
// may instead be pre-seeded (e.g. by a Vault static secret). It is intentionally
// not owned/overwritten once it exists.
func (r *KeaAPIReconciler) reconcileTokenSecret(ctx context.Context, api *v1alpha1.KeaAPI) error {
name := tokenSecretName(api)
var existing corev1.Secret
err := r.Get(ctx, types.NamespacedName{Namespace: api.Namespace, Name: name}, &existing)
if err == nil {
return nil
}
if !apierrors.IsNotFound(err) {
return err
}
tok := make([]byte, 32)
if _, err := rand.Read(tok); err != nil {
return err
}
sec := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: api.Namespace, Labels: apiLabels(api.Name)},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{"token": hex.EncodeToString(tok)},
}
return r.Create(ctx, sec)
}
func (r *KeaAPIReconciler) reconcileDeployment(ctx context.Context, api *v1alpha1.KeaAPI) error {
image := api.Spec.Image
if image == "" {
image = defaultAPIImage
}
replicas := int32(1)
if api.Spec.Replicas != nil {
replicas = *api.Spec.Replicas
}
port := api.Spec.Service.Port
if port == 0 {
port = 8080
}
dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: api.Name, Namespace: api.Namespace}}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, dep, func() error {
dep.Labels = apiLabels(api.Name)
dep.Spec.Replicas = int32ptr(replicas)
dep.Spec.Selector = &metav1.LabelSelector{MatchLabels: apiLabels(api.Name)}
dep.Spec.Template = corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: apiLabels(api.Name)},
Spec: corev1.PodSpec{
ServiceAccountName: api.Name,
NodeSelector: api.Spec.NodeSelector,
Tolerations: api.Spec.Tolerations,
Affinity: api.Spec.Affinity,
Containers: []corev1.Container{{
Name: "kea-api",
Image: image,
Command: []string{"kea-api"},
Ports: []corev1.ContainerPort{{Name: "http", ContainerPort: port}},
Env: []corev1.EnvVar{
{Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", port)},
{Name: "TARGET_NAMESPACE", Value: api.Namespace},
{Name: "KEA_API_TOKEN", ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: tokenSecretName(api)},
Key: "token",
},
}},
},
Resources: api.Spec.Resources,
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz", Port: intstrFromInt(int(port)),
}},
InitialDelaySeconds: 3, PeriodSeconds: 10,
},
SecurityContext: &corev1.SecurityContext{
RunAsNonRoot: ptr(true),
AllowPrivilegeEscalation: ptr(false),
ReadOnlyRootFilesystem: ptr(true),
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
},
}},
},
}
return ctrl.SetControllerReference(api, dep, r.Scheme)
})
return err
}
func (r *KeaAPIReconciler) reconcileService(ctx context.Context, api *v1alpha1.KeaAPI) error {
port := api.Spec.Service.Port
if port == 0 {
port = 8080
}
svcType := api.Spec.Service.Type
if svcType == "" {
svcType = corev1.ServiceTypeClusterIP
}
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: api.Name, Namespace: api.Namespace}}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, svc, func() error {
svc.Labels = apiLabels(api.Name)
svc.Annotations = api.Spec.Service.Annotations
svc.Spec.Type = svcType
svc.Spec.Selector = apiLabels(api.Name)
svc.Spec.Ports = []corev1.ServicePort{{Name: "http", Port: port, TargetPort: intstrFromInt(int(port))}}
return ctrl.SetControllerReference(api, svc, r.Scheme)
})
return err
}
func tokenSecretName(api *v1alpha1.KeaAPI) string {
if api.Spec.TokenSecretName != "" {
return api.Spec.TokenSecretName
}
return api.Name + "-token"
}
func apiLabels(name string) map[string]string {
return map[string]string{
managedByLabel: managedByValue,
"app.kubernetes.io/name": "kea-api",
"app.kubernetes.io/instance": name,
}
}
func (r *KeaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.KeaAPI{}).
Owns(&appsv1.Deployment{}).
Owns(&corev1.Service{}).
Owns(&corev1.ServiceAccount{}).
Owns(&rbacv1.Role{}).
Owns(&rbacv1.RoleBinding{}).
Complete(r)
}
@@ -0,0 +1,57 @@
package controller
import (
"context"
"strings"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
)
// KeaClientClassReconciler validates KeaClientClass CRs and maintains status.
type KeaClientClassReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaclientclasses,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaclientclasses/status,verbs=get;update;patch
func (r *KeaClientClassReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var cc v1alpha1.KeaClientClass
if err := r.Get(ctx, req.NamespacedName, &cc); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if cc.Spec.Test == "" && len(cc.Spec.ArchHex) == 0 {
cc.Status.Phase = "Invalid"
setReady(&cc.Status.Conditions, cc.Generation, false, "NoMatch", "either test or archHex must be set")
_ = r.Status().Update(ctx, &cc)
return ctrl.Result{}, nil
}
for _, a := range cc.Spec.ArchHex {
if !strings.HasPrefix(a, "0x") {
cc.Status.Phase = "Invalid"
setReady(&cc.Status.Conditions, cc.Generation, false, "BadArch", "archHex values must be 0x-prefixed, e.g. 0x0007")
_ = r.Status().Update(ctx, &cc)
return ctrl.Result{}, nil
}
}
cc.Status.Phase = "Ready"
cc.Status.ObservedGeneration = cc.Generation
setReady(&cc.Status.Conditions, cc.Generation, true, "Validated", "client class accepted")
if err := r.Status().Update(ctx, &cc); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
func (r *KeaClientClassReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.KeaClientClass{}).
Complete(r)
}
@@ -0,0 +1,396 @@
package controller
import (
"context"
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
"git.unkin.net/unkin/kea-operator/internal/kea"
)
// KeaClusterReconciler reconciles a KeaCluster.
type KeaClusterReconciler struct {
client.Client
Scheme *runtime.Scheme
Control *kea.ControlClient
}
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaclusters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaclusters/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keasubnets,verbs=get;list;watch
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keaclientclasses,verbs=get;list;watch
// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=services;configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
func (r *KeaClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
var cluster v1alpha1.KeaCluster
if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if err := r.reconcileConfigMap(ctx, &cluster); err != nil {
return r.fail(ctx, &cluster, "ConfigError", err)
}
if err := r.reconcileServices(ctx, &cluster); err != nil {
return r.fail(ctx, &cluster, "ServiceError", err)
}
sts, err := r.reconcileStatefulSet(ctx, &cluster)
if err != nil {
return r.fail(ctx, &cluster, "WorkloadError", err)
}
r.reloadReadyPods(ctx, &cluster)
ready := sts.Status.ReadyReplicas
desired := int32(1)
if cluster.Spec.Replicas != nil {
desired = *cluster.Spec.Replicas
}
cluster.Status.ObservedGeneration = cluster.Generation
cluster.Status.Replicas = sts.Status.Replicas
cluster.Status.ReadyReplicas = ready
cluster.Status.ServiceIP = r.serviceIP(ctx, &cluster)
if ready > 0 {
cluster.Status.ActivePeer = cluster.Name + "-0"
}
if ready >= desired && desired > 0 {
cluster.Status.Phase = "Ready"
setReady(&cluster.Status.Conditions, cluster.Generation, true, "Ready", "all replicas ready")
} else {
cluster.Status.Phase = "Progressing"
setReady(&cluster.Status.Conditions, cluster.Generation, false, "Progressing",
fmt.Sprintf("%d/%d replicas ready", ready, desired))
}
if err := r.Status().Update(ctx, &cluster); err != nil {
l.Error(err, "status update")
}
if cluster.Status.Phase != "Ready" {
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
return ctrl.Result{RequeueAfter: requeueLong}, nil
}
func (r *KeaClusterReconciler) fail(ctx context.Context, c *v1alpha1.KeaCluster, reason string, err error) (ctrl.Result, error) {
c.Status.Phase = "Error"
setReady(&c.Status.Conditions, c.Generation, false, reason, err.Error())
_ = r.Status().Update(ctx, c)
return ctrl.Result{}, err
}
// buildInput gathers matching subnets/classes and the stable HA peer list.
func (r *KeaClusterReconciler) buildInput(ctx context.Context, c *v1alpha1.KeaCluster) (kea.RenderInput, error) {
var subnetList v1alpha1.KeaSubnetList
if err := r.List(ctx, &subnetList, client.InNamespace(c.Namespace)); err != nil {
return kea.RenderInput{}, err
}
var subnets []v1alpha1.KeaSubnet
for _, s := range subnetList.Items {
if s.Spec.ClusterRef == "" || s.Spec.ClusterRef == c.Name {
subnets = append(subnets, s)
}
}
var classList v1alpha1.KeaClientClassList
if err := r.List(ctx, &classList, client.InNamespace(c.Namespace)); err != nil {
return kea.RenderInput{}, err
}
var classes []v1alpha1.KeaClientClass
for _, cc := range classList.Items {
if cc.Spec.ClusterRef == "" || cc.Spec.ClusterRef == c.Name {
classes = append(classes, cc)
}
}
return kea.RenderInput{
Cluster: *c,
Subnets: subnets,
ClientClasses: classes,
Peers: r.peers(c),
}, nil
}
// peers returns stable HA peer identities (DNS only, no pod IPs).
func (r *KeaClusterReconciler) peers(c *v1alpha1.KeaCluster) []kea.Peer {
replicas := int32(1)
if c.Spec.Replicas != nil {
replicas = *c.Spec.Replicas
}
mode := c.Spec.HA.Mode
if mode == "" {
mode = v1alpha1.HAHotStandby
}
peers := make([]kea.Peer, 0, replicas)
for i := int32(0); i < replicas; i++ {
role := "backup"
switch {
case i == 0:
role = "primary"
case i == 1 && mode == v1alpha1.HAHotStandby:
role = "standby"
case i == 1:
role = "secondary"
}
peers = append(peers, kea.Peer{
Name: fmt.Sprintf("server%d", i),
URL: peerDNS(c.Name, c.Namespace, int(i)),
Role: role,
})
}
return peers
}
func (r *KeaClusterReconciler) reconcileConfigMap(ctx context.Context, c *v1alpha1.KeaCluster) error {
in, err := r.buildInput(ctx, c)
if err != nil {
return err
}
dhcp4, err := kea.RenderDHCP4(in)
if err != nil {
return err
}
agent, err := kea.RenderCtrlAgent()
if err != nil {
return err
}
cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: configMapName(c.Name), Namespace: c.Namespace}}
_, err = ctrl.CreateOrUpdate(ctx, r.Client, cm, func() error {
cm.Labels = commonLabels(c.Name)
cm.Data = map[string]string{
"kea-dhcp4.conf": dhcp4,
"kea-ctrl-agent.conf": agent,
"entrypoint-dhcp4.sh": kea.EntrypointDHCP4(),
"entrypoint-ctrlagent.sh": kea.EntrypointCtrlAgent(),
}
return ctrl.SetControllerReference(c, cm, r.Scheme)
})
return err
}
func (r *KeaClusterReconciler) reconcileServices(ctx context.Context, c *v1alpha1.KeaCluster) error {
// Headless service for stable per-pod DNS (HA peer URLs, ctrl-agent).
headless := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: headlessName(c.Name), Namespace: c.Namespace}}
if _, err := ctrl.CreateOrUpdate(ctx, r.Client, headless, func() error {
headless.Labels = commonLabels(c.Name)
headless.Spec.ClusterIP = corev1.ClusterIPNone
headless.Spec.PublishNotReadyAddresses = true
headless.Spec.Selector = commonLabels(c.Name)
headless.Spec.Ports = []corev1.ServicePort{
{Name: "ctrl", Port: kea.CtrlAgentPort, Protocol: corev1.ProtocolTCP},
}
return ctrl.SetControllerReference(c, headless, r.Scheme)
}); err != nil {
return err
}
// Anycast DHCP service (LoadBalancer via PureLB by default).
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName(c.Name), Namespace: c.Namespace}}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, svc, func() error {
svc.Labels = commonLabels(c.Name)
svc.Annotations = mergeAnnotations(c.Spec.Service)
svc.Spec.Selector = commonLabels(c.Name)
svcType := c.Spec.Service.Type
if svcType == "" {
svcType = corev1.ServiceTypeLoadBalancer
}
svc.Spec.Type = svcType
if c.Spec.Service.LoadBalancerIP != "" {
svc.Spec.LoadBalancerIP = c.Spec.Service.LoadBalancerIP
}
if c.Spec.Service.LoadBalancerClass != nil {
svc.Spec.LoadBalancerClass = c.Spec.Service.LoadBalancerClass
}
if svcType == corev1.ServiceTypeLoadBalancer || svcType == corev1.ServiceTypeNodePort {
svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyLocal
}
svc.Spec.Ports = []corev1.ServicePort{
{Name: "dhcp", Port: kea.DHCP4Port, Protocol: corev1.ProtocolUDP},
}
return ctrl.SetControllerReference(c, svc, r.Scheme)
})
return err
}
func mergeAnnotations(s v1alpha1.ClusterServiceSpec) map[string]string {
out := map[string]string{}
for k, v := range s.Annotations {
out[k] = v
}
if s.IPAddressPool != "" {
out["purelb.io/service-group"] = s.IPAddressPool
}
if len(out) == 0 {
return nil
}
return out
}
func (r *KeaClusterReconciler) reconcileStatefulSet(ctx context.Context, c *v1alpha1.KeaCluster) (*appsv1.StatefulSet, error) {
hash, err := configHash(ctx, r.Client, c.Namespace, configMapName(c.Name))
if err != nil {
return nil, err
}
image := c.Spec.Image
if image == "" {
image = kea.DefaultImage
}
replicas := int32(1)
if c.Spec.Replicas != nil {
replicas = *c.Spec.Replicas
}
sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: stsName(c.Name), Namespace: c.Namespace}}
_, err = ctrl.CreateOrUpdate(ctx, r.Client, sts, func() error {
sts.Labels = commonLabels(c.Name)
sts.Spec.ServiceName = headlessName(c.Name)
sts.Spec.Replicas = int32ptr(replicas)
sts.Spec.Selector = &metav1.LabelSelector{MatchLabels: commonLabels(c.Name)}
sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
sts.Spec.Template = r.podTemplate(c, image, hash)
return ctrl.SetControllerReference(c, sts, r.Scheme)
})
if err != nil {
return nil, err
}
return sts, nil
}
func (r *KeaClusterReconciler) podTemplate(c *v1alpha1.KeaCluster, image, hash string) corev1.PodTemplateSpec {
volProjected := corev1.Volume{
Name: "kea-etc",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{Name: configMapName(c.Name)},
DefaultMode: int32ptr(0o755),
},
},
}
volRun := corev1.Volume{Name: "run", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}
mounts := []corev1.VolumeMount{
{Name: "kea-etc", MountPath: kea.ConfigDir, ReadOnly: true},
{Name: "run", MountPath: kea.RunDir},
}
dhcp4 := corev1.Container{
Name: kea.ContainerDHCP4,
Image: image,
Command: []string{"/bin/sh", kea.ConfigDir + "/entrypoint-dhcp4.sh"},
Resources: c.Spec.Resources,
Ports: []corev1.ContainerPort{
{Name: "dhcp", ContainerPort: kea.DHCP4Port, Protocol: corev1.ProtocolUDP},
},
VolumeMounts: mounts,
}
agent := corev1.Container{
Name: kea.ContainerCtrlAgent,
Image: image,
Command: []string{"/bin/sh", kea.ConfigDir + "/entrypoint-ctrlagent.sh"},
Resources: c.Spec.Resources,
Ports: []corev1.ContainerPort{
{Name: "ctrl", ContainerPort: kea.CtrlAgentPort, Protocol: corev1.ProtocolTCP},
},
VolumeMounts: mounts,
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt(kea.CtrlAgentPort)}},
InitialDelaySeconds: 5, PeriodSeconds: 10,
},
}
labels := commonLabels(c.Name)
return corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Annotations: map[string]string{"kea.unkin.net/config-hash": hash},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{dhcp4, agent},
Volumes: []corev1.Volume{volProjected, volRun},
NodeSelector: c.Spec.NodeSelector,
Tolerations: c.Spec.Tolerations,
Affinity: c.Spec.Affinity,
},
}
}
// reloadReadyPods best-effort hot-reloads config on ready pods via the
// ctrl-agent REST channel, analogous to bind-operator's rndc reconfig.
func (r *KeaClusterReconciler) reloadReadyPods(ctx context.Context, c *v1alpha1.KeaCluster) {
if r.Control == nil {
return
}
var pods corev1.PodList
if err := r.List(ctx, &pods, client.InNamespace(c.Namespace), client.MatchingLabels(commonLabels(c.Name))); err != nil {
return
}
l := log.FromContext(ctx)
for i := range pods.Items {
p := &pods.Items[i]
if p.Status.PodIP == "" || !podReady(p) {
continue
}
url := fmt.Sprintf("http://%s:%d/", p.Status.PodIP, kea.CtrlAgentPort)
if err := r.Control.ConfigReload(ctx, url); err != nil {
l.V(1).Info("config-reload failed", "pod", p.Name, "err", err.Error())
}
}
}
func (r *KeaClusterReconciler) serviceIP(ctx context.Context, c *v1alpha1.KeaCluster) string {
var svc corev1.Service
if err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: serviceName(c.Name)}, &svc); err != nil {
return ""
}
if len(svc.Status.LoadBalancer.Ingress) > 0 {
return svc.Status.LoadBalancer.Ingress[0].IP
}
return svc.Spec.ClusterIP
}
func podReady(p *corev1.Pod) bool {
for _, cond := range p.Status.Conditions {
if cond.Type == corev1.PodReady {
return cond.Status == corev1.ConditionTrue
}
}
return false
}
func (r *KeaClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
mapToClusters := func(ctx context.Context, obj client.Object) []reconcile.Request {
var list v1alpha1.KeaClusterList
if err := r.List(ctx, &list, client.InNamespace(obj.GetNamespace())); err != nil {
return nil
}
var reqs []reconcile.Request
for _, c := range list.Items {
reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: c.Namespace, Name: c.Name}})
}
return reqs
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.KeaCluster{}).
Owns(&appsv1.StatefulSet{}).
Owns(&corev1.Service{}).
Owns(&corev1.ConfigMap{}).
Watches(&v1alpha1.KeaSubnet{}, handler.EnqueueRequestsFromMapFunc(mapToClusters)).
Watches(&v1alpha1.KeaClientClass{}, handler.EnqueueRequestsFromMapFunc(mapToClusters)).
Complete(r)
}
+167
View File
@@ -0,0 +1,167 @@
package controller
import (
"context"
"testing"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
)
func testScheme(t *testing.T) *runtime.Scheme {
t.Helper()
s := runtime.NewScheme()
if err := clientgoscheme.AddToScheme(s); err != nil {
t.Fatal(err)
}
if err := v1alpha1.AddToScheme(s); err != nil {
t.Fatal(err)
}
return s
}
func newClusterFixture() *v1alpha1.KeaCluster {
return &v1alpha1.KeaCluster{
ObjectMeta: metav1.ObjectMeta{Name: "pxe", Namespace: "dhcp-system"},
Spec: v1alpha1.KeaClusterSpec{
Replicas: int32ptr(2),
DomainName: "main.unkin.net",
HA: v1alpha1.HASpec{Mode: v1alpha1.HAHotStandby},
},
}
}
func TestKeaClusterReconcileCreatesWorkload(t *testing.T) {
scheme := testScheme(t)
cluster := newClusterFixture()
subnet := &v1alpha1.KeaSubnet{
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", Pools: []string{"198.18.13.200 - 198.18.13.220"}},
}
cl := fake.NewClientBuilder().
WithScheme(scheme).
WithStatusSubresource(&v1alpha1.KeaCluster{}).
WithObjects(cluster, subnet).
Build()
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
t.Fatalf("reconcile: %v", err)
}
// ConfigMap rendered with the subnet.
var cm corev1.ConfigMap
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm); err != nil {
t.Fatalf("configmap not created: %v", err)
}
if got := cm.Data["kea-dhcp4.conf"]; got == "" || !contains(got, "198.18.13.0/24") {
t.Errorf("configmap missing subnet render")
}
// StatefulSet with 2 containers and a config-hash annotation.
var sts appsv1.StatefulSet
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
t.Fatalf("statefulset not created: %v", err)
}
if *sts.Spec.Replicas != 2 {
t.Errorf("expected 2 replicas, got %d", *sts.Spec.Replicas)
}
if len(sts.Spec.Template.Spec.Containers) != 2 {
t.Errorf("expected kea-dhcp4 + kea-ctrl-agent containers, got %d", len(sts.Spec.Template.Spec.Containers))
}
if sts.Spec.Template.Annotations["kea.unkin.net/config-hash"] == "" {
t.Errorf("missing config-hash annotation")
}
// Anycast + headless services.
for _, name := range []string{"pxe", "pxe-headless"} {
var svc corev1.Service
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: name}, &svc); err != nil {
t.Errorf("service %s not created: %v", name, err)
}
}
}
// TestConfigHashChangesWithSubnets guards the roll trigger: adding a subnet
// must change the pod-template config hash (so the STS rolls).
func TestConfigHashChangesWithSubnets(t *testing.T) {
scheme := testScheme(t)
hashFor := func(objs ...client.Object) string {
base := []client.Object{newClusterFixture()}
cl := fake.NewClientBuilder().
WithScheme(scheme).
WithStatusSubresource(&v1alpha1.KeaCluster{}).
WithObjects(append(base, objs...)...).
Build()
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
t.Fatal(err)
}
var sts appsv1.StatefulSet
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}, &sts); err != nil {
t.Fatal(err)
}
return sts.Spec.Template.Annotations["kea.unkin.net/config-hash"]
}
empty := hashFor()
withSubnet := hashFor(&v1alpha1.KeaSubnet{
ObjectMeta: metav1.ObjectMeta{Name: "s13", Namespace: "dhcp-system"},
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24"},
})
if empty == withSubnet {
t.Errorf("config hash did not change when a subnet was added")
}
}
func TestClusterRefFiltersSubnets(t *testing.T) {
scheme := testScheme(t)
cluster := newClusterFixture()
mine := &v1alpha1.KeaSubnet{
ObjectMeta: metav1.ObjectMeta{Name: "mine", Namespace: "dhcp-system"},
Spec: v1alpha1.KeaSubnetSpec{Subnet: "198.18.13.0/24", ClusterRef: "pxe"},
}
other := &v1alpha1.KeaSubnet{
ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "dhcp-system"},
Spec: v1alpha1.KeaSubnetSpec{Subnet: "10.9.9.0/24", ClusterRef: "someone-else"},
}
cl := fake.NewClientBuilder().WithScheme(scheme).
WithStatusSubresource(&v1alpha1.KeaCluster{}).
WithObjects(cluster, mine, other).Build()
r := &KeaClusterReconciler{Client: cl, Scheme: scheme}
if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "dhcp-system", Name: "pxe"}}); err != nil {
t.Fatal(err)
}
var cm corev1.ConfigMap
_ = cl.Get(context.Background(), types.NamespacedName{Namespace: "dhcp-system", Name: "pxe-config"}, &cm)
conf := cm.Data["kea-dhcp4.conf"]
if !contains(conf, "198.18.13.0/24") {
t.Errorf("cluster-matched subnet missing from config")
}
if contains(conf, "10.9.9.0/24") {
t.Errorf("subnet bound to another cluster leaked into config")
}
}
func contains(hay, needle string) bool {
return len(hay) >= len(needle) && (indexOf(hay, needle) >= 0)
}
func indexOf(hay, needle string) int {
for i := 0; i+len(needle) <= len(hay); i++ {
if hay[i:i+len(needle)] == needle {
return i
}
}
return -1
}
@@ -0,0 +1,72 @@
package controller
import (
"context"
"fmt"
"net"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
v1alpha1 "git.unkin.net/unkin/kea-operator/api/v1alpha1"
)
// KeaSubnetReconciler validates KeaSubnet CRs and maintains their status. The
// actual kea config is rendered by the KeaCluster controller, which watches
// subnets and re-renders on change.
type KeaSubnetReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keasubnets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=kea.unkin.net,resources=keasubnets/status,verbs=get;update;patch
func (r *KeaSubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var subnet v1alpha1.KeaSubnet
if err := r.Get(ctx, req.NamespacedName, &subnet); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if _, _, err := net.ParseCIDR(subnet.Spec.Subnet); err != nil {
subnet.Status.Phase = "Invalid"
setReady(&subnet.Status.Conditions, subnet.Generation, false, "InvalidCIDR", err.Error())
_ = r.Status().Update(ctx, &subnet)
return ctrl.Result{}, nil
}
for _, p := range subnet.Spec.Pools {
if err := validatePool(p); err != nil {
subnet.Status.Phase = "Invalid"
setReady(&subnet.Status.Conditions, subnet.Generation, false, "InvalidPool", err.Error())
_ = r.Status().Update(ctx, &subnet)
return ctrl.Result{}, nil
}
}
subnet.Status.Phase = "Ready"
subnet.Status.AssignedID = subnet.Spec.ID
subnet.Status.ObservedGeneration = subnet.Generation
setReady(&subnet.Status.Conditions, subnet.Generation, true, "Validated", "subnet accepted")
if err := r.Status().Update(ctx, &subnet); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
func validatePool(p string) error {
// Accept "start - end" (with or without spaces) or a single address.
fields := splitPool(p)
for _, f := range fields {
if net.ParseIP(f) == nil {
return fmt.Errorf("invalid pool address %q in %q", f, p)
}
}
return nil
}
func (r *KeaSubnetReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.KeaSubnet{}).
Complete(r)
}
+37
View File
@@ -0,0 +1,37 @@
package controller
import (
ctrl "sigs.k8s.io/controller-runtime"
"git.unkin.net/unkin/kea-operator/internal/kea"
)
// SetupAll wires every reconciler into the manager.
func SetupAll(mgr ctrl.Manager) error {
if err := (&KeaClusterReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Control: kea.NewControlClient(),
}).SetupWithManager(mgr); err != nil {
return err
}
if err := (&KeaSubnetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
return err
}
if err := (&KeaClientClassReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
return err
}
if err := (&KeaAPIReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
return err
}
return nil
}