Implement autobackup-operator controllers, tests, CI and packaging
PVCs and CloudNativePG Clusters need S3 buckets and backup schedules
provisioned consistently. This operator watches the
backups.unkin.net/{schedule,destination} annotations on those objects and
provisions everything needed to back them up, with no new CRDs.
- Add a PVC controller that provisions cephrgw ObjectStoreUser/Bucket/BucketAccess,
auto-generates a restic repo-password Secret and creates a k8up Schedule scoped
to the PVC via spec.backup.volumes[].persistentVolumeClaim.claimName.
- Add a CNPG Cluster controller that provisions the same bucket stack, idempotently
patches spec.backup.barmanObjectStore (leaving a user-set destinationPath alone
with a Warning event) and creates a ScheduledBackup.
- Resolve destinations through a ConfigMap lookup table; requeue until the
BucketAccess is Ready before creating schedule resources; own-reference created
resources and retain bucket data by default.
- Add schedule-mapping helpers (k8up 5-field/shortcut pass-through, CNPG 6-field
seconds-first) and deterministic, length-bounded name derivation.
- Add unit tests (schedule mapping, name derivation, destination resolution) and
envtest controller tests for both paths, wiring the external CRDs into envtest.
- Add kubebuilder-generated RBAC, a Dockerfile (distroless/nonroot), Woodpecker
lint/test/build pipelines and a tag-triggered image push to the artifactapi
docker-internal registry, plus a version-bump Makefile and deploy manifests.
This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// CNPG typed API is deliberately not imported: its module graph (barman-cloud,
|
||||
// cnpg-i/gRPC, prometheus-operator) is heavy to vendor, so Clusters and
|
||||
// ScheduledBackups are handled as unstructured objects.
|
||||
var (
|
||||
clusterGVK = schema.GroupVersionKind{Group: "postgresql.cnpg.io", Version: "v1", Kind: "Cluster"}
|
||||
scheduledBackupGVK = schema.GroupVersionKind{Group: "postgresql.cnpg.io", Version: "v1", Kind: "ScheduledBackup"}
|
||||
)
|
||||
|
||||
func newCluster() *unstructured.Unstructured {
|
||||
u := &unstructured.Unstructured{}
|
||||
u.SetGroupVersionKind(clusterGVK)
|
||||
return u
|
||||
}
|
||||
|
||||
func newScheduledBackup() *unstructured.Unstructured {
|
||||
u := &unstructured.Unstructured{}
|
||||
u.SetGroupVersionKind(scheduledBackupGVK)
|
||||
return u
|
||||
}
|
||||
|
||||
// ClusterReconciler backs up annotated CloudNativePG Clusters by provisioning an
|
||||
// S3 bucket (via cephrgw), patching spec.backup.barmanObjectStore and creating a
|
||||
// ScheduledBackup.
|
||||
type ClusterReconciler struct {
|
||||
baseReconciler
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=clusters,verbs=get;list;watch;update;patch
|
||||
// +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=scheduledbackups,verbs=get;list;watch;create;update;patch;delete
|
||||
|
||||
func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
cluster := newCluster()
|
||||
if err := r.Get(ctx, req.NamespacedName, cluster); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
schedule := cluster.GetAnnotations()[annSchedule]
|
||||
if schedule == "" {
|
||||
return r.teardown(ctx, cluster)
|
||||
}
|
||||
|
||||
cnpgSchedule, err := ScheduleForCNPG(schedule)
|
||||
if err != nil {
|
||||
r.Recorder.Event(cluster, corev1.EventTypeWarning, "InvalidSchedule", err.Error())
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
destName := cluster.GetAnnotations()[annDestination]
|
||||
if destName == "" {
|
||||
r.Recorder.Event(cluster, corev1.EventTypeWarning, "MissingDestination", "backups.unkin.net/destination annotation is required")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
dest, err := r.resolveDest(ctx, destName)
|
||||
if err != nil {
|
||||
if errors.Is(err, errDestinationNotFound) {
|
||||
r.Recorder.Event(cluster, corev1.EventTypeWarning, "UnknownDestination", err.Error())
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
ready, credSecret, err := r.ensureBucketStack(ctx, cluster, dest)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if !ready {
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
if err := r.patchBarman(ctx, cluster, credSecret, dest); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if err := r.ensureScheduledBackup(ctx, cluster, cnpgSchedule); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
logger.Info("cluster backup reconciled", "cluster", cluster.GetName(), "destination", destName)
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
func (r *ClusterReconciler) teardown(ctx context.Context, cluster *unstructured.Unstructured) (ctrl.Result, error) {
|
||||
sb := newScheduledBackup()
|
||||
sb.SetName(scheduleName(cluster.GetName()))
|
||||
sb.SetNamespace(cluster.GetNamespace())
|
||||
return ctrl.Result{}, deleteIfExists(ctx, r.Client, sb)
|
||||
}
|
||||
|
||||
// patchBarman idempotently sets spec.backup.barmanObjectStore on the Cluster. If
|
||||
// the user already set a different destinationPath it is left untouched and a
|
||||
// Warning event is emitted instead of clobbering it.
|
||||
func (r *ClusterReconciler) patchBarman(ctx context.Context, cluster *unstructured.Unstructured, credSecret string, dest Destination) error {
|
||||
bucket := bucketName(cluster.GetNamespace(), cluster.GetName())
|
||||
desiredPath := "s3://" + bucket
|
||||
|
||||
existing, _, _ := unstructured.NestedString(cluster.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
|
||||
if existing != "" && existing != desiredPath {
|
||||
r.Recorder.Eventf(cluster, corev1.EventTypeWarning, "DestinationConflict",
|
||||
"cluster already configured with destinationPath %q; not overwriting", existing)
|
||||
return nil
|
||||
}
|
||||
|
||||
patch := client.MergeFrom(cluster.DeepCopy())
|
||||
|
||||
barman := map[string]interface{}{
|
||||
"destinationPath": desiredPath,
|
||||
"endpointURL": dest.Endpoint,
|
||||
"serverName": cluster.GetName(),
|
||||
"s3Credentials": map[string]interface{}{
|
||||
"accessKeyId": map[string]interface{}{"name": credSecret, "key": "AWS_ACCESS_KEY_ID"},
|
||||
"secretAccessKey": map[string]interface{}{"name": credSecret, "key": "AWS_SECRET_ACCESS_KEY"},
|
||||
},
|
||||
}
|
||||
if dest.EndpointCASecret != "" {
|
||||
key := dest.EndpointCAKey
|
||||
if key == "" {
|
||||
key = "ca.crt"
|
||||
}
|
||||
barman["endpointCA"] = map[string]interface{}{"name": dest.EndpointCASecret, "key": key}
|
||||
}
|
||||
if err := unstructured.SetNestedMap(cluster.Object, barman, "spec", "backup", "barmanObjectStore"); err != nil {
|
||||
return err
|
||||
}
|
||||
if rp, _, _ := unstructured.NestedString(cluster.Object, "spec", "backup", "retentionPolicy"); rp == "" {
|
||||
if err := unstructured.SetNestedField(cluster.Object, "30d", "spec", "backup", "retentionPolicy"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return r.Patch(ctx, cluster, patch)
|
||||
}
|
||||
|
||||
func (r *ClusterReconciler) ensureScheduledBackup(ctx context.Context, cluster *unstructured.Unstructured, schedule string) error {
|
||||
sb := newScheduledBackup()
|
||||
sb.SetName(scheduleName(cluster.GetName()))
|
||||
sb.SetNamespace(cluster.GetNamespace())
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, sb, func() error {
|
||||
for _, f := range []struct {
|
||||
value string
|
||||
path []string
|
||||
}{
|
||||
{cluster.GetName(), []string{"spec", "cluster", "name"}},
|
||||
{schedule, []string{"spec", "schedule"}},
|
||||
{"barmanObjectStore", []string{"spec", "method"}},
|
||||
{"self", []string{"spec", "backupOwnerReference"}},
|
||||
} {
|
||||
if err := unstructured.SetNestedField(sb.Object, f.value, f.path...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return controllerutil.SetControllerReference(cluster, sb, r.Scheme)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(newCluster()).
|
||||
Owns(newScheduledBackup()).
|
||||
Named("cluster-autobackup").
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
func TestClusterReconcile(t *testing.T) {
|
||||
requireEnvtest(t)
|
||||
ctx := context.Background()
|
||||
ns := newNamespace(t, ctx)
|
||||
createDestinations(t, ctx, ns)
|
||||
|
||||
cluster := newCluster()
|
||||
cluster.SetName("pg")
|
||||
cluster.SetNamespace(ns)
|
||||
cluster.SetAnnotations(map[string]string{
|
||||
annSchedule: "@daily",
|
||||
annDestination: "cephs3_ec4_1",
|
||||
})
|
||||
_ = unstructured.SetNestedField(cluster.Object, int64(1), "spec", "instances")
|
||||
if err := k8sClient.Create(ctx, cluster); err != nil {
|
||||
t.Fatalf("create cluster: %v", err)
|
||||
}
|
||||
|
||||
r := &ClusterReconciler{baseReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: clientgoscheme.Scheme,
|
||||
Recorder: record.NewFakeRecorder(16),
|
||||
DestNamespace: ns,
|
||||
DestConfigMap: "autobackup-destinations",
|
||||
}}
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "pg"}}
|
||||
|
||||
// First reconcile provisions the bucket stack but blocks on access readiness.
|
||||
if _, err := r.Reconcile(ctx, req); err != nil {
|
||||
t.Fatalf("first reconcile: %v", err)
|
||||
}
|
||||
var access cephv1.BucketAccess
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("pg")}, &access); err != nil {
|
||||
t.Fatalf("BucketAccess not created: %v", err)
|
||||
}
|
||||
access.Status.Phase = "Ready"
|
||||
access.Status.SecretName = credSecretName("pg")
|
||||
if err := k8sClient.Status().Update(ctx, &access); err != nil {
|
||||
t.Fatalf("update access status: %v", err)
|
||||
}
|
||||
|
||||
if _, err := r.Reconcile(ctx, req); err != nil {
|
||||
t.Fatalf("second reconcile: %v", err)
|
||||
}
|
||||
|
||||
// Cluster barmanObjectStore patched.
|
||||
got := newCluster()
|
||||
if err := k8sClient.Get(ctx, req.NamespacedName, got); err != nil {
|
||||
t.Fatalf("get cluster: %v", err)
|
||||
}
|
||||
path, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
|
||||
if want := "s3://" + bucketName(ns, "pg"); path != want {
|
||||
t.Errorf("destinationPath = %q, want %q", path, want)
|
||||
}
|
||||
endpoint, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "endpointURL")
|
||||
if endpoint != "https://s3.ceph.unkin.net" {
|
||||
t.Errorf("endpointURL = %q", endpoint)
|
||||
}
|
||||
server, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "serverName")
|
||||
if server != "pg" {
|
||||
t.Errorf("serverName = %q", server)
|
||||
}
|
||||
akName, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "s3Credentials", "accessKeyId", "name")
|
||||
if akName != credSecretName("pg") {
|
||||
t.Errorf("accessKeyId secret name = %q", akName)
|
||||
}
|
||||
caName, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "endpointCA", "name")
|
||||
if caName != "vault-ca-cert" {
|
||||
t.Errorf("endpointCA name = %q", caName)
|
||||
}
|
||||
rp, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "retentionPolicy")
|
||||
if rp != "30d" {
|
||||
t.Errorf("retentionPolicy = %q", rp)
|
||||
}
|
||||
|
||||
// ScheduledBackup created with 6-field schedule.
|
||||
sb := newScheduledBackup()
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("pg")}, sb); err != nil {
|
||||
t.Fatalf("ScheduledBackup not created: %v", err)
|
||||
}
|
||||
sched, _, _ := unstructured.NestedString(sb.Object, "spec", "schedule")
|
||||
if sched != "0 0 0 * * *" {
|
||||
t.Errorf("scheduledbackup schedule = %q, want 0 0 0 * * *", sched)
|
||||
}
|
||||
method, _, _ := unstructured.NestedString(sb.Object, "spec", "method")
|
||||
if method != "barmanObjectStore" {
|
||||
t.Errorf("method = %q", method)
|
||||
}
|
||||
clusterName, _, _ := unstructured.NestedString(sb.Object, "spec", "cluster", "name")
|
||||
if clusterName != "pg" {
|
||||
t.Errorf("cluster.name = %q", clusterName)
|
||||
}
|
||||
owner, _, _ := unstructured.NestedString(sb.Object, "spec", "backupOwnerReference")
|
||||
if owner != "self" {
|
||||
t.Errorf("backupOwnerReference = %q", owner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterReconcileDestinationConflict(t *testing.T) {
|
||||
requireEnvtest(t)
|
||||
ctx := context.Background()
|
||||
ns := newNamespace(t, ctx)
|
||||
createDestinations(t, ctx, ns)
|
||||
|
||||
cluster := newCluster()
|
||||
cluster.SetName("pg2")
|
||||
cluster.SetNamespace(ns)
|
||||
cluster.SetAnnotations(map[string]string{annSchedule: "@daily", annDestination: "cephs3_ec4_1"})
|
||||
_ = unstructured.SetNestedField(cluster.Object, int64(1), "spec", "instances")
|
||||
// User already configured a different destinationPath.
|
||||
_ = unstructured.SetNestedField(cluster.Object, "s3://user-owned-bucket", "spec", "backup", "barmanObjectStore", "destinationPath")
|
||||
if err := k8sClient.Create(ctx, cluster); err != nil {
|
||||
t.Fatalf("create cluster: %v", err)
|
||||
}
|
||||
|
||||
rec := record.NewFakeRecorder(16)
|
||||
r := &ClusterReconciler{baseReconciler{
|
||||
Client: k8sClient, Scheme: clientgoscheme.Scheme, Recorder: rec,
|
||||
DestNamespace: ns, DestConfigMap: "autobackup-destinations",
|
||||
}}
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "pg2"}}
|
||||
|
||||
if _, err := r.Reconcile(ctx, req); err != nil {
|
||||
t.Fatalf("first reconcile: %v", err)
|
||||
}
|
||||
var access cephv1.BucketAccess
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("pg2")}, &access); err != nil {
|
||||
t.Fatalf("BucketAccess not created: %v", err)
|
||||
}
|
||||
access.Status.Phase = "Ready"
|
||||
access.Status.SecretName = credSecretName("pg2")
|
||||
if err := k8sClient.Status().Update(ctx, &access); err != nil {
|
||||
t.Fatalf("update access status: %v", err)
|
||||
}
|
||||
if _, err := r.Reconcile(ctx, req); err != nil {
|
||||
t.Fatalf("second reconcile: %v", err)
|
||||
}
|
||||
|
||||
got := newCluster()
|
||||
if err := k8sClient.Get(ctx, req.NamespacedName, got); err != nil {
|
||||
t.Fatalf("get cluster: %v", err)
|
||||
}
|
||||
path, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
|
||||
if path != "s3://user-owned-bucket" {
|
||||
t.Errorf("operator clobbered user destinationPath: got %q", path)
|
||||
}
|
||||
|
||||
// ScheduledBackup is still created; only the barman patch is skipped.
|
||||
sb := newScheduledBackup()
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("pg2")}, sb); err != nil {
|
||||
t.Fatalf("ScheduledBackup should still be created: %v", err)
|
||||
}
|
||||
|
||||
var sawConflict bool
|
||||
for drained := false; !drained; {
|
||||
select {
|
||||
case e := <-rec.Events:
|
||||
if strings.Contains(e, "DestinationConflict") {
|
||||
sawConflict = true
|
||||
}
|
||||
default:
|
||||
drained = true
|
||||
}
|
||||
}
|
||||
if !sawConflict {
|
||||
t.Errorf("expected a DestinationConflict warning event")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
// Destination is a resolved backup target parsed from the operator's
|
||||
// destinations ConfigMap.
|
||||
type Destination struct {
|
||||
PlacementTarget string `json:"placementTarget,omitempty"`
|
||||
Zonegroup string `json:"zonegroup,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
EndpointCASecret string `json:"endpointCASecret,omitempty"`
|
||||
EndpointCAKey string `json:"endpointCAKey,omitempty"`
|
||||
}
|
||||
|
||||
// errDestinationNotFound signals a missing ConfigMap or entry so callers can
|
||||
// emit a Warning event and requeue instead of treating it as a hard failure.
|
||||
var errDestinationNotFound = errors.New("destination not found")
|
||||
|
||||
// resolveDestination looks a logical destination name up in the operator's
|
||||
// destinations ConfigMap and parses its YAML entry.
|
||||
func resolveDestination(ctx context.Context, c client.Client, ns, cmName, dest string) (Destination, error) {
|
||||
var cm corev1.ConfigMap
|
||||
if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: cmName}, &cm); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return Destination{}, fmt.Errorf("destinations ConfigMap %s/%s: %w", ns, cmName, errDestinationNotFound)
|
||||
}
|
||||
return Destination{}, err
|
||||
}
|
||||
raw, ok := cm.Data[dest]
|
||||
if !ok {
|
||||
return Destination{}, fmt.Errorf("destination %q: %w", dest, errDestinationNotFound)
|
||||
}
|
||||
var d Destination
|
||||
if err := yaml.Unmarshal([]byte(raw), &d); err != nil {
|
||||
return Destination{}, fmt.Errorf("destination %q: %w", dest, err)
|
||||
}
|
||||
if d.Endpoint == "" {
|
||||
return Destination{}, fmt.Errorf("destination %q: missing endpoint", dest)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
func destClient(t *testing.T, objs ...runtime.Object) *fake.ClientBuilder {
|
||||
t.Helper()
|
||||
return fake.NewClientBuilder().WithScheme(clientgoscheme.Scheme).WithRuntimeObjects(objs...)
|
||||
}
|
||||
|
||||
func TestResolveDestinationHit(t *testing.T) {
|
||||
cm := &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
|
||||
Data: map[string]string{
|
||||
"cephs3_ec4_1": "placementTarget: cephs3_ec4_1\nzonegroup: au\nendpoint: https://s3.ceph.unkin.net\nendpointCASecret: vault-ca-cert\nendpointCAKey: ca.crt\n",
|
||||
},
|
||||
}
|
||||
c := destClient(t, cm).Build()
|
||||
d, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if d.PlacementTarget != "cephs3_ec4_1" || d.Zonegroup != "au" || d.Endpoint != "https://s3.ceph.unkin.net" {
|
||||
t.Errorf("unexpected destination: %+v", d)
|
||||
}
|
||||
if d.EndpointCASecret != "vault-ca-cert" || d.EndpointCAKey != "ca.crt" {
|
||||
t.Errorf("unexpected CA fields: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDestinationMissEntry(t *testing.T) {
|
||||
cm := &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
|
||||
Data: map[string]string{"other": "endpoint: https://x\n"},
|
||||
}
|
||||
c := destClient(t, cm).Build()
|
||||
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
|
||||
if !errors.Is(err, errDestinationNotFound) {
|
||||
t.Errorf("expected errDestinationNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDestinationMissConfigMap(t *testing.T) {
|
||||
c := destClient(t).Build()
|
||||
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
|
||||
if !errors.Is(err, errDestinationNotFound) {
|
||||
t.Errorf("expected errDestinationNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDestinationMissingEndpoint(t *testing.T) {
|
||||
cm := &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
|
||||
Data: map[string]string{"noendpoint": "placementTarget: x\n"},
|
||||
}
|
||||
c := destClient(t, cm).Build()
|
||||
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "noendpoint")
|
||||
if err == nil {
|
||||
t.Errorf("expected error for missing endpoint")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
)
|
||||
|
||||
const (
|
||||
annPrefix = "backups.unkin.net/"
|
||||
annSchedule = annPrefix + "schedule"
|
||||
annDestination = annPrefix + "destination"
|
||||
annPurge = annPrefix + "purge-on-delete"
|
||||
|
||||
// requeueSteady re-syncs healthy objects so drift heals without waiting for
|
||||
// an event.
|
||||
requeueSteady = 10 * time.Minute
|
||||
// requeueShort backs off while waiting for a dependency (e.g. the
|
||||
// BucketAccess to become Ready or a destination to appear).
|
||||
requeueShort = 30 * time.Second
|
||||
)
|
||||
|
||||
// baseReconciler holds the shared client, scheme, recorder and destinations
|
||||
// ConfigMap location used by both the PVC and CNPG Cluster controllers.
|
||||
type baseReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Recorder record.EventRecorder
|
||||
DestNamespace string
|
||||
DestConfigMap string
|
||||
}
|
||||
|
||||
func (b *baseReconciler) resolveDest(ctx context.Context, dest string) (Destination, error) {
|
||||
return resolveDestination(ctx, b.Client, b.DestNamespace, b.DestConfigMap, dest)
|
||||
}
|
||||
|
||||
// ensureBucketStack provisions the ObjectStoreUser, Bucket and BucketAccess for
|
||||
// an annotated object (all owner-referenced to it) and reports whether the
|
||||
// BucketAccess is Ready together with the name of its credential Secret.
|
||||
func (b *baseReconciler) ensureBucketStack(ctx context.Context, owner client.Object, dest Destination) (ready bool, credSecret string, err error) {
|
||||
ns := owner.GetNamespace()
|
||||
obj := owner.GetName()
|
||||
bName := bucketName(ns, obj)
|
||||
|
||||
user := &cephv1.ObjectStoreUser{ObjectMeta: metav1.ObjectMeta{Name: userName(obj), Namespace: ns}}
|
||||
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, user, func() error {
|
||||
user.Spec.DisplayName = fmt.Sprintf("autobackup %s/%s", ns, obj)
|
||||
return controllerutil.SetControllerReference(owner, user, b.Scheme)
|
||||
}); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
bucket := &cephv1.Bucket{ObjectMeta: metav1.ObjectMeta{Name: bName, Namespace: ns}}
|
||||
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, bucket, func() error {
|
||||
bucket.Spec.OwnerRef = user.Name
|
||||
bucket.Spec.BucketName = bName
|
||||
bucket.Spec.PlacementTarget = dest.PlacementTarget
|
||||
bucket.Spec.Zonegroup = dest.Zonegroup
|
||||
bucket.Spec.Versioning = true
|
||||
// Keep bucket data by default; deleting the owner GCs the CRs but the
|
||||
// underlying objects are retained for safety.
|
||||
bucket.Spec.RetainOnDelete = true
|
||||
return controllerutil.SetControllerReference(owner, bucket, b.Scheme)
|
||||
}); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
access := &cephv1.BucketAccess{ObjectMeta: metav1.ObjectMeta{Name: accessName(obj), Namespace: ns}}
|
||||
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, access, func() error {
|
||||
access.Spec.BucketRef = bName
|
||||
access.Spec.Level = cephv1.AccessReadWrite
|
||||
access.Spec.SecretName = credSecretName(obj)
|
||||
return controllerutil.SetControllerReference(owner, access, b.Scheme)
|
||||
}); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
if access.Status.Phase != "Ready" || access.Status.SecretName == "" {
|
||||
return false, "", nil
|
||||
}
|
||||
return true, access.Status.SecretName, nil
|
||||
}
|
||||
|
||||
// ensureResticSecret creates (once) an owner-referenced Secret holding a random
|
||||
// restic repository password under key "password".
|
||||
func (b *baseReconciler) ensureResticSecret(ctx context.Context, owner client.Object) (string, error) {
|
||||
name := resticSecretName(owner.GetName())
|
||||
sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: owner.GetNamespace()}}
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, b.Client, sec, func() error {
|
||||
sec.Type = corev1.SecretTypeOpaque
|
||||
if _, ok := sec.Data["password"]; !ok {
|
||||
pw, err := randomPassword(32)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sec.Data == nil {
|
||||
sec.Data = map[string][]byte{}
|
||||
}
|
||||
sec.Data["password"] = []byte(pw)
|
||||
}
|
||||
return controllerutil.SetControllerReference(owner, sec, b.Scheme)
|
||||
})
|
||||
return name, err
|
||||
}
|
||||
|
||||
func randomPassword(n int) (string, error) {
|
||||
buf := make([]byte, n)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// deleteIfExists best-effort deletes obj, ignoring a NotFound result.
|
||||
func deleteIfExists(ctx context.Context, c client.Client, obj client.Object) error {
|
||||
return client.IgnoreNotFound(c.Delete(ctx, obj))
|
||||
}
|
||||
|
||||
// bucketStackObjects lists the cephrgw CRs and credential Secret created for an
|
||||
// object, for explicit purge-on-delete teardown.
|
||||
func bucketStackObjects(ns, obj string) []client.Object {
|
||||
meta := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Name: name, Namespace: ns} }
|
||||
return []client.Object{
|
||||
&cephv1.BucketAccess{ObjectMeta: meta(accessName(obj))},
|
||||
&cephv1.Bucket{ObjectMeta: meta(bucketName(ns, obj))},
|
||||
&cephv1.ObjectStoreUser{ObjectMeta: meta(userName(obj))},
|
||||
&corev1.Secret{ObjectMeta: meta(credSecretName(obj))},
|
||||
&corev1.Secret{ObjectMeta: meta(resticSecretName(obj))},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxObjectName is the DNS-1123 subdomain limit for Kubernetes object names.
|
||||
maxObjectName = 253
|
||||
// maxBucketName is the DNS-1123 label limit RGW enforces on S3 bucket names.
|
||||
maxBucketName = 63
|
||||
namePrefix = "autobk"
|
||||
)
|
||||
|
||||
var invalidNameChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
||||
|
||||
// bucketName derives the namespace-scoped, globally-unique S3 bucket name (and
|
||||
// the Bucket CR name) for an annotated object, clamped to the 63-char RGW limit.
|
||||
func bucketName(namespace, obj string) string {
|
||||
return boundedName(maxBucketName, namePrefix, namespace, obj)
|
||||
}
|
||||
|
||||
// userName, accessName, credSecretName, resticSecretName and scheduleName derive
|
||||
// the per-object child resource names. These are namespaced objects, so the
|
||||
// namespace is not part of the name (only uniqueness within the namespace is
|
||||
// required).
|
||||
func userName(obj string) string { return boundedName(maxObjectName, namePrefix, obj, "owner") }
|
||||
func accessName(obj string) string { return boundedName(maxObjectName, namePrefix, obj, "rw") }
|
||||
func credSecretName(obj string) string { return boundedName(maxObjectName, obj, "autobackup", "rgw") }
|
||||
func resticSecretName(obj string) string {
|
||||
return boundedName(maxObjectName, obj, "autobackup", "restic")
|
||||
}
|
||||
func scheduleName(obj string) string { return boundedName(maxObjectName, obj, "autobackup") }
|
||||
|
||||
// boundedName joins parts with "-", sanitises to a DNS-safe token and, when the
|
||||
// result would exceed max, truncates it and appends a short deterministic hash
|
||||
// of the full joined value so long inputs stay unique and within limits.
|
||||
func boundedName(max int, parts ...string) string {
|
||||
joined := strings.Join(parts, "-")
|
||||
s := sanitizeName(joined)
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
sum := sha1.Sum([]byte(joined))
|
||||
h := hex.EncodeToString(sum[:])[:8]
|
||||
keep := max - len(h) - 1
|
||||
if keep < 1 {
|
||||
return h[:max]
|
||||
}
|
||||
return strings.TrimRight(s[:keep], "-") + "-" + h
|
||||
}
|
||||
|
||||
func sanitizeName(s string) string {
|
||||
s = strings.ToLower(s)
|
||||
s = invalidNameChars.ReplaceAllString(s, "-")
|
||||
s = strings.Trim(s, "-")
|
||||
if s == "" {
|
||||
return namePrefix
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDerivedNames(t *testing.T) {
|
||||
if got := bucketName("prod", "data"); got != "autobk-prod-data" {
|
||||
t.Errorf("bucketName = %q", got)
|
||||
}
|
||||
if got := userName("data"); got != "autobk-data-owner" {
|
||||
t.Errorf("userName = %q", got)
|
||||
}
|
||||
if got := accessName("data"); got != "autobk-data-rw" {
|
||||
t.Errorf("accessName = %q", got)
|
||||
}
|
||||
if got := credSecretName("data"); got != "data-autobackup-rgw" {
|
||||
t.Errorf("credSecretName = %q", got)
|
||||
}
|
||||
if got := resticSecretName("data"); got != "data-autobackup-restic" {
|
||||
t.Errorf("resticSecretName = %q", got)
|
||||
}
|
||||
if got := scheduleName("data"); got != "data-autobackup" {
|
||||
t.Errorf("scheduleName = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketNameSanitisesAndClamps(t *testing.T) {
|
||||
// Upper-case and invalid characters are lowercased/replaced.
|
||||
if got := bucketName("Prod_NS", "My.Data"); strings.ToLower(got) != got {
|
||||
t.Errorf("bucketName not lowercased: %q", got)
|
||||
}
|
||||
|
||||
longNS := strings.Repeat("a", 40)
|
||||
longObj := strings.Repeat("b", 40)
|
||||
got := bucketName(longNS, longObj)
|
||||
if len(got) > maxBucketName {
|
||||
t.Errorf("bucketName %q exceeds %d chars (%d)", got, maxBucketName, len(got))
|
||||
}
|
||||
// Deterministic: same inputs -> same hashed name.
|
||||
if got2 := bucketName(longNS, longObj); got != got2 {
|
||||
t.Errorf("bucketName not deterministic: %q vs %q", got, got2)
|
||||
}
|
||||
// Distinct long inputs -> distinct names (hash disambiguates).
|
||||
if other := bucketName(longNS, strings.Repeat("c", 40)); other == got {
|
||||
t.Errorf("bucketName collision for distinct inputs: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectNameClamps(t *testing.T) {
|
||||
longObj := strings.Repeat("x", 300)
|
||||
got := scheduleName(longObj)
|
||||
if len(got) > maxObjectName {
|
||||
t.Errorf("scheduleName %q exceeds %d chars (%d)", got, maxObjectName, len(got))
|
||||
}
|
||||
if got2 := scheduleName(longObj); got != got2 {
|
||||
t.Errorf("scheduleName not deterministic")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// PVCReconciler backs up annotated PersistentVolumeClaims by provisioning an S3
|
||||
// bucket (via cephrgw) and a k8up Schedule scoped to the PVC.
|
||||
type PVCReconciler struct {
|
||||
baseReconciler
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch
|
||||
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers;buckets;bucketaccesses,verbs=get;list;watch;create;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/status,verbs=get
|
||||
// +kubebuilder:rbac:groups=k8up.io,resources=schedules,verbs=get;list;watch;create;update;patch;delete
|
||||
|
||||
func (r *PVCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var pvc corev1.PersistentVolumeClaim
|
||||
if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
schedule := pvc.Annotations[annSchedule]
|
||||
if schedule == "" {
|
||||
return r.teardown(ctx, &pvc)
|
||||
}
|
||||
|
||||
k8upSchedule, err := ScheduleForK8up(schedule)
|
||||
if err != nil {
|
||||
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "InvalidSchedule", err.Error())
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
destName := pvc.Annotations[annDestination]
|
||||
if destName == "" {
|
||||
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "MissingDestination", "backups.unkin.net/destination annotation is required")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
dest, err := r.resolveDest(ctx, destName)
|
||||
if err != nil {
|
||||
if errors.Is(err, errDestinationNotFound) {
|
||||
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "UnknownDestination", err.Error())
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
ready, credSecret, err := r.ensureBucketStack(ctx, &pvc, dest)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if !ready {
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
resticSecret, err := r.ensureResticSecret(ctx, &pvc)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if err := r.ensureSchedule(ctx, &pvc, k8upSchedule, credSecret, resticSecret, dest); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
logger.Info("pvc backup reconciled", "pvc", pvc.Name, "destination", destName)
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
// teardown removes the k8up Schedule when the annotation is gone, and (only on
|
||||
// explicit opt-in) the bucket stack. Bucket data is retained by default.
|
||||
func (r *PVCReconciler) teardown(ctx context.Context, pvc *corev1.PersistentVolumeClaim) (ctrl.Result, error) {
|
||||
if err := deleteIfExists(ctx, r.Client, &k8upv1.Schedule{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: scheduleName(pvc.Name), Namespace: pvc.Namespace},
|
||||
}); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if !strings.EqualFold(pvc.Annotations[annPurge], "true") {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
for _, obj := range bucketStackObjects(pvc.Namespace, pvc.Name) {
|
||||
if err := deleteIfExists(ctx, r.Client, obj); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *PVCReconciler) ensureSchedule(ctx context.Context, pvc *corev1.PersistentVolumeClaim, schedule, credSecret, resticSecret string, dest Destination) error {
|
||||
sched := &k8upv1.Schedule{ObjectMeta: metav1.ObjectMeta{Name: scheduleName(pvc.Name), Namespace: pvc.Namespace}}
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, sched, func() error {
|
||||
sched.Spec.Backend = &k8upv1.Backend{
|
||||
RepoPasswordSecretRef: secretKeyRef(resticSecret, "password"),
|
||||
S3: &k8upv1.S3Spec{
|
||||
Endpoint: dest.Endpoint,
|
||||
Bucket: bucketName(pvc.Namespace, pvc.Name),
|
||||
AccessKeyIDSecretRef: secretKeyRef(credSecret, "AWS_ACCESS_KEY_ID"),
|
||||
SecretAccessKeySecretRef: secretKeyRef(credSecret, "AWS_SECRET_ACCESS_KEY"),
|
||||
},
|
||||
}
|
||||
sched.Spec.Backup = &k8upv1.BackupSchedule{
|
||||
ScheduleCommon: &k8upv1.ScheduleCommon{Schedule: k8upv1.ScheduleDefinition(schedule)},
|
||||
BackupSpec: k8upv1.BackupSpec{
|
||||
RunnableSpec: k8upv1.RunnableSpec{
|
||||
Volumes: &[]k8upv1.RunnableVolumeSpec{{
|
||||
Name: "backup-source",
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
sched.Spec.Prune = &k8upv1.PruneSchedule{
|
||||
ScheduleCommon: &k8upv1.ScheduleCommon{Schedule: k8upv1.ScheduleDefinition("@weekly")},
|
||||
PruneSpec: k8upv1.PruneSpec{Retention: k8upv1.RetentionPolicy{KeepDaily: 7, KeepWeekly: 4}},
|
||||
}
|
||||
return controllerutil.SetControllerReference(pvc, sched, r.Scheme)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func secretKeyRef(name, key string) *corev1.SecretKeySelector {
|
||||
return &corev1.SecretKeySelector{
|
||||
LocalObjectReference: corev1.LocalObjectReference{Name: name},
|
||||
Key: key,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *PVCReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&corev1.PersistentVolumeClaim{}).
|
||||
Owns(&k8upv1.Schedule{}).
|
||||
Named("pvc-autobackup").
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
func TestPVCReconcile(t *testing.T) {
|
||||
requireEnvtest(t)
|
||||
ctx := context.Background()
|
||||
ns := newNamespace(t, ctx)
|
||||
createDestinations(t, ctx, ns)
|
||||
|
||||
pvc := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "data",
|
||||
Namespace: ns,
|
||||
Annotations: map[string]string{
|
||||
annSchedule: "@daily",
|
||||
annDestination: "cephs3_ec4_1",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.VolumeResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")},
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := k8sClient.Create(ctx, pvc); err != nil {
|
||||
t.Fatalf("create pvc: %v", err)
|
||||
}
|
||||
|
||||
r := &PVCReconciler{baseReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: clientgoscheme.Scheme,
|
||||
Recorder: record.NewFakeRecorder(16),
|
||||
DestNamespace: ns,
|
||||
DestConfigMap: "autobackup-destinations",
|
||||
}}
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "data"}}
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("first reconcile: %v", err)
|
||||
}
|
||||
if res.RequeueAfter != requeueShort {
|
||||
t.Errorf("expected requeueShort while access not ready, got %v", res.RequeueAfter)
|
||||
}
|
||||
|
||||
// Bucket stack was created and wired.
|
||||
var user cephv1.ObjectStoreUser
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: userName("data")}, &user); err != nil {
|
||||
t.Fatalf("ObjectStoreUser not created: %v", err)
|
||||
}
|
||||
var bucket cephv1.Bucket
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: bucketName(ns, "data")}, &bucket); err != nil {
|
||||
t.Fatalf("Bucket not created: %v", err)
|
||||
}
|
||||
if bucket.Spec.OwnerRef != userName("data") || !bucket.Spec.Versioning || bucket.Spec.PlacementTarget != "cephs3_ec4_1" {
|
||||
t.Errorf("unexpected bucket spec: %+v", bucket.Spec)
|
||||
}
|
||||
if !bucket.Spec.RetainOnDelete {
|
||||
t.Errorf("expected RetainOnDelete true for data safety")
|
||||
}
|
||||
var access cephv1.BucketAccess
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("data")}, &access); err != nil {
|
||||
t.Fatalf("BucketAccess not created: %v", err)
|
||||
}
|
||||
if access.Spec.Level != cephv1.AccessReadWrite || access.Spec.SecretName != credSecretName("data") {
|
||||
t.Errorf("unexpected access spec: %+v", access.Spec)
|
||||
}
|
||||
|
||||
// No Schedule until the BucketAccess is Ready.
|
||||
var sched k8upv1.Schedule
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("data")}, &sched); !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("Schedule should not exist before access Ready, got err=%v", err)
|
||||
}
|
||||
|
||||
// Simulate cephrgw marking the access Ready.
|
||||
access.Status.Phase = "Ready"
|
||||
access.Status.SecretName = credSecretName("data")
|
||||
if err := k8sClient.Status().Update(ctx, &access); err != nil {
|
||||
t.Fatalf("update access status: %v", err)
|
||||
}
|
||||
|
||||
res, err = r.Reconcile(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("second reconcile: %v", err)
|
||||
}
|
||||
if res.RequeueAfter != requeueSteady {
|
||||
t.Errorf("expected requeueSteady after ready, got %v", res.RequeueAfter)
|
||||
}
|
||||
|
||||
// Restic password secret exists with a password.
|
||||
var restic corev1.Secret
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: resticSecretName("data")}, &restic); err != nil {
|
||||
t.Fatalf("restic secret not created: %v", err)
|
||||
}
|
||||
if len(restic.Data["password"]) == 0 {
|
||||
t.Errorf("restic secret has empty password")
|
||||
}
|
||||
|
||||
// Schedule created and wired to the cephrgw + restic secrets.
|
||||
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("data")}, &sched); err != nil {
|
||||
t.Fatalf("Schedule not created: %v", err)
|
||||
}
|
||||
if sched.Spec.Backend == nil || sched.Spec.Backend.S3 == nil {
|
||||
t.Fatalf("Schedule backend not set: %+v", sched.Spec)
|
||||
}
|
||||
s3 := sched.Spec.Backend.S3
|
||||
if s3.Bucket != bucketName(ns, "data") || s3.Endpoint != "https://s3.ceph.unkin.net" {
|
||||
t.Errorf("unexpected S3 backend: %+v", s3)
|
||||
}
|
||||
if s3.AccessKeyIDSecretRef.Name != credSecretName("data") || s3.AccessKeyIDSecretRef.Key != "AWS_ACCESS_KEY_ID" {
|
||||
t.Errorf("unexpected access key ref: %+v", s3.AccessKeyIDSecretRef)
|
||||
}
|
||||
if sched.Spec.Backend.RepoPasswordSecretRef.Name != resticSecretName("data") {
|
||||
t.Errorf("unexpected repo password ref: %+v", sched.Spec.Backend.RepoPasswordSecretRef)
|
||||
}
|
||||
if sched.Spec.Backup == nil || string(sched.Spec.Backup.Schedule) != "@daily" {
|
||||
t.Errorf("unexpected backup schedule: %+v", sched.Spec.Backup)
|
||||
}
|
||||
vols := sched.Spec.Backup.Volumes
|
||||
if vols == nil || len(*vols) != 1 || (*vols)[0].PersistentVolumeClaim == nil || (*vols)[0].PersistentVolumeClaim.ClaimName != "data" {
|
||||
t.Errorf("expected schedule scoped to PVC data, got %+v", vols)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// cnpgShortcuts expands cron nickname shortcuts to the 6-field (seconds-first)
|
||||
// cron expressions CloudNativePG's robfig/cron parser expects.
|
||||
var cnpgShortcuts = map[string]string{
|
||||
"@hourly": "0 0 * * * *",
|
||||
"@daily": "0 0 0 * * *",
|
||||
"@midnight": "0 0 0 * * *",
|
||||
"@weekly": "0 0 0 * * 0",
|
||||
"@monthly": "0 0 0 1 * *",
|
||||
"@yearly": "0 0 0 1 1 *",
|
||||
"@annually": "0 0 0 1 1 *",
|
||||
}
|
||||
|
||||
// k8upShortcuts is the set of cron nicknames k8up accepts natively (passed
|
||||
// through unchanged).
|
||||
var k8upShortcuts = map[string]bool{
|
||||
"@hourly": true, "@daily": true, "@midnight": true, "@weekly": true,
|
||||
"@monthly": true, "@yearly": true, "@annually": true,
|
||||
}
|
||||
|
||||
// ScheduleForK8up validates a schedule annotation for k8up, which accepts cron
|
||||
// nickname shortcuts natively and otherwise expects a 5-field cron expression.
|
||||
// A valid value is returned unchanged.
|
||||
func ScheduleForK8up(in string) (string, error) {
|
||||
s := strings.TrimSpace(in)
|
||||
if s == "" {
|
||||
return "", fmt.Errorf("empty schedule")
|
||||
}
|
||||
if strings.HasPrefix(s, "@") {
|
||||
if k8upShortcuts[s] {
|
||||
return s, nil
|
||||
}
|
||||
return "", fmt.Errorf("unsupported schedule shortcut %q", s)
|
||||
}
|
||||
if err := validateCronFields(s, 5); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ScheduleForCNPG normalises a schedule annotation into the 6-field
|
||||
// (seconds-first) cron format a CloudNativePG ScheduledBackup requires:
|
||||
// shortcuts expand to explicit expressions, a 5-field cron gains a leading "0"
|
||||
// seconds field, and a 6-field cron is accepted as-is.
|
||||
func ScheduleForCNPG(in string) (string, error) {
|
||||
s := strings.TrimSpace(in)
|
||||
if s == "" {
|
||||
return "", fmt.Errorf("empty schedule")
|
||||
}
|
||||
if strings.HasPrefix(s, "@") {
|
||||
if v, ok := cnpgShortcuts[s]; ok {
|
||||
return v, nil
|
||||
}
|
||||
return "", fmt.Errorf("unsupported schedule shortcut %q", s)
|
||||
}
|
||||
fields := strings.Fields(s)
|
||||
switch len(fields) {
|
||||
case 5:
|
||||
if err := validateCronFields(s, 5); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "0 " + strings.Join(fields, " "), nil
|
||||
case 6:
|
||||
if err := validateCronFields(s, 6); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.Join(fields, " "), nil
|
||||
default:
|
||||
return "", fmt.Errorf("cron %q must have 5 or 6 fields, got %d", s, len(fields))
|
||||
}
|
||||
}
|
||||
|
||||
func validateCronFields(s string, want int) error {
|
||||
fields := strings.Fields(s)
|
||||
if len(fields) != want {
|
||||
return fmt.Errorf("cron %q must have %d fields, got %d", s, want, len(fields))
|
||||
}
|
||||
for _, f := range fields {
|
||||
if f == "" {
|
||||
return fmt.Errorf("cron %q has an empty field", s)
|
||||
}
|
||||
for _, r := range f {
|
||||
if !isCronRune(r) {
|
||||
return fmt.Errorf("cron field %q contains invalid character %q", f, string(r))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isCronRune(r rune) bool {
|
||||
return (r >= '0' && r <= '9') || r == '*' || r == '/' || r == ',' || r == '-' || r == '?'
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package controller
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestScheduleForK8up(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"@hourly", "@hourly", false},
|
||||
{"@daily", "@daily", false},
|
||||
{"@weekly", "@weekly", false},
|
||||
{"@monthly", "@monthly", false},
|
||||
{" @daily ", "@daily", false},
|
||||
{"0 2 * * *", "0 2 * * *", false},
|
||||
{"*/15 * * * *", "*/15 * * * *", false},
|
||||
{"0 0 * * 1-5", "0 0 * * 1-5", false},
|
||||
{"@bogus", "", true},
|
||||
{"0 2 * *", "", true}, // 4 fields
|
||||
{"0 0 0 * * *", "", true}, // 6 fields not valid for k8up
|
||||
{"0 2 * * foo", "", true}, // invalid chars
|
||||
{"", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := ScheduleForK8up(c.in)
|
||||
if c.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ScheduleForK8up(%q) expected error, got %q", c.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("ScheduleForK8up(%q) unexpected error: %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("ScheduleForK8up(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleForCNPG(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"@hourly", "0 0 * * * *", false},
|
||||
{"@daily", "0 0 0 * * *", false},
|
||||
{"@midnight", "0 0 0 * * *", false},
|
||||
{"@weekly", "0 0 0 * * 0", false},
|
||||
{"@monthly", "0 0 0 1 * *", false},
|
||||
{"@yearly", "0 0 0 1 1 *", false},
|
||||
{"@annually", "0 0 0 1 1 *", false},
|
||||
{"0 2 * * *", "0 0 2 * * *", false}, // 5-field gains seconds
|
||||
{"*/15 * * * *", "0 */15 * * * *", false},
|
||||
{"30 0 3 * * *", "30 0 3 * * *", false}, // already 6-field
|
||||
{" @daily ", "0 0 0 * * *", false},
|
||||
{"@bogus", "", true},
|
||||
{"0 2 * *", "", true}, // 4 fields
|
||||
{"1 2 3 4 5 6 7", "", true}, // 7 fields
|
||||
{"0 2 * * bad", "", true}, // invalid chars
|
||||
{"", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := ScheduleForCNPG(c.in)
|
||||
if c.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ScheduleForCNPG(%q) expected error, got %q", c.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("ScheduleForCNPG(%q) unexpected error: %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("ScheduleForCNPG(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
)
|
||||
|
||||
// SetupAll registers both controllers with the manager.
|
||||
func SetupAll(mgr ctrl.Manager, destNamespace, destConfigMap string) error {
|
||||
base := func(recorder string) baseReconciler {
|
||||
return baseReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Recorder: mgr.GetEventRecorderFor(recorder),
|
||||
DestNamespace: destNamespace,
|
||||
DestConfigMap: destConfigMap,
|
||||
}
|
||||
}
|
||||
if err := (&PVCReconciler{baseReconciler: base("autobackup-pvc")}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := (&ClusterReconciler{baseReconciler: base("autobackup-cluster")}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
)
|
||||
|
||||
var (
|
||||
testEnv *envtest.Environment
|
||||
k8sClient client.Client
|
||||
envtestReady bool
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
utilruntime.Must(cephv1.AddToScheme(clientgoscheme.Scheme))
|
||||
utilruntime.Must(k8upv1.AddToScheme(clientgoscheme.Scheme))
|
||||
|
||||
testEnv = &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "..", "test", "crds")},
|
||||
ErrorIfCRDPathMissing: true,
|
||||
}
|
||||
cfg, err := testEnv.Start()
|
||||
if err != nil {
|
||||
// envtest binaries (KUBEBUILDER_ASSETS) may be unavailable; run the
|
||||
// pure-unit tests and skip the controller tests.
|
||||
fmt.Fprintf(os.Stderr, "envtest unavailable, skipping controller tests: %v\n", err)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: clientgoscheme.Scheme})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to build client: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
envtestReady = true
|
||||
code := m.Run()
|
||||
_ = testEnv.Stop()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func requireEnvtest(t *testing.T) {
|
||||
t.Helper()
|
||||
if !envtestReady {
|
||||
t.Skip("envtest not available (KUBEBUILDER_ASSETS unset)")
|
||||
}
|
||||
}
|
||||
|
||||
func newNamespace(t *testing.T, ctx context.Context) string {
|
||||
t.Helper()
|
||||
name := fmt.Sprintf("test-%s", randSuffix())
|
||||
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}
|
||||
if err := k8sClient.Create(ctx, ns); err != nil {
|
||||
t.Fatalf("create namespace: %v", err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func createDestinations(t *testing.T, ctx context.Context, ns string) {
|
||||
t.Helper()
|
||||
cm := &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: ns},
|
||||
Data: map[string]string{
|
||||
"cephs3_ec4_1": "placementTarget: cephs3_ec4_1\nzonegroup: au\nendpoint: https://s3.ceph.unkin.net\nendpointCASecret: vault-ca-cert\nendpointCAKey: ca.crt\n",
|
||||
},
|
||||
}
|
||||
if err := k8sClient.Create(ctx, cm); err != nil {
|
||||
t.Fatalf("create destinations configmap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var suffixCounter int
|
||||
|
||||
func randSuffix() string {
|
||||
suffixCounter++
|
||||
return fmt.Sprintf("%d", suffixCounter)
|
||||
}
|
||||
Reference in New Issue
Block a user