Initial cephrgw-operator: Ceph RGW buckets & keys via dashboard API
Adds a Kubernetes operator that provisions Ceph RGW (S3) buckets and access keys declaratively through the Ceph manager dashboard REST API. Three CRDs in group ceph.unkin.net/v1alpha1: - ObjectStoreUser: creates an RGW user, delivers its key pair to a Secret - Bucket: creates an S3 bucket owned by an ObjectStoreUser; owns the bucket's aggregate S3 policy (union of all BucketAccess grants) - BucketAccess: grants read-only/read-write/full access, provisioning a dedicated user (or reusing a referenced one) and delivering RW/RO keys The internal/ceph client wraps the dashboard /api/auth, /api/rgw/user and /api/rgw/bucket endpoints with lazy token auth and re-auth on 401. Bucket policies are rendered deterministically and applied via the bucket policy API (Reef 18.2+). Credentials come from the cephrgw-credentials Secret via env. Includes generated CRDs/RBAC, samples, kind manifests, Woodpecker CI, and docs/ceph-setup.md covering the required Ceph dashboard account, RGW wiring and permissions.
This commit is contained in:
benvin
@@ -0,0 +1,246 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// BucketReconciler provisions RGW buckets and owns the bucket's S3 policy. It
|
||||
// aggregates every BucketAccess that targets the bucket into a single policy
|
||||
// document, so the policy stays convergent no matter the order of events.
|
||||
type BucketReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/finalizers,verbs=update
|
||||
|
||||
func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var b v1alpha1.Bucket
|
||||
if err := r.Get(ctx, req.NamespacedName, &b); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
bucketName := orDefault(b.Spec.BucketName, b.Name)
|
||||
|
||||
if !b.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&b, finalizer) {
|
||||
if !b.Spec.RetainOnDelete {
|
||||
if err := r.Ceph.DeleteBucket(ctx, bucketName, b.Spec.PurgeOnDelete); err != nil {
|
||||
return r.fail(ctx, &b, "DeleteFailed", err)
|
||||
}
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&b, finalizer)
|
||||
if err := r.Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&b, finalizer) {
|
||||
if err := r.Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the owning user.
|
||||
var owner v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Spec.OwnerRef}, &owner); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.pending(ctx, &b, "OwnerMissing", fmt.Sprintf("waiting for ObjectStoreUser %q", b.Spec.OwnerRef))
|
||||
}
|
||||
return r.fail(ctx, &b, "OwnerLookupFailed", err)
|
||||
}
|
||||
if owner.Status.UID == "" || owner.Status.Phase != "Ready" {
|
||||
return r.pending(ctx, &b, "OwnerNotReady", fmt.Sprintf("ObjectStoreUser %q not ready", b.Spec.OwnerRef))
|
||||
}
|
||||
ownerUID := owner.Status.UID
|
||||
|
||||
// Ensure the bucket exists.
|
||||
info, err := r.Ceph.GetBucket(ctx, bucketName)
|
||||
if ceph.IsNotFound(err) {
|
||||
createSpec := ceph.CreateBucketSpec{
|
||||
Bucket: bucketName,
|
||||
OwnerUID: ownerUID,
|
||||
Zonegroup: b.Spec.Zonegroup,
|
||||
PlacementTarget: b.Spec.PlacementTarget,
|
||||
}
|
||||
if ol := b.Spec.ObjectLock; ol != nil && ol.Enabled {
|
||||
createSpec.LockEnabled = true
|
||||
createSpec.LockMode = string(ol.Mode)
|
||||
createSpec.LockDays = ol.Days
|
||||
createSpec.LockYears = ol.Years
|
||||
}
|
||||
info, err = r.Ceph.CreateBucket(ctx, createSpec)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "CreateFailed", err)
|
||||
}
|
||||
logger.Info("created bucket", "bucket", bucketName, "owner", ownerUID)
|
||||
} else if err != nil {
|
||||
return r.fail(ctx, &b, "LookupFailed", err)
|
||||
}
|
||||
bucketID := info.InstanceID()
|
||||
|
||||
// Versioning (forced on when object lock is enabled).
|
||||
if b.Spec.Versioning || (b.Spec.ObjectLock != nil && b.Spec.ObjectLock.Enabled) {
|
||||
if err := r.Ceph.SetBucketVersioning(ctx, bucketName, bucketID, ownerUID, true); err != nil {
|
||||
return r.fail(ctx, &b, "VersioningFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tags.
|
||||
if len(b.Spec.Tags) > 0 {
|
||||
tj, err := ceph.BuildTagJSON(b.Spec.Tags)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "TagsFailed", err)
|
||||
}
|
||||
if tj != "" {
|
||||
if err := r.Ceph.SetBucketTags(ctx, bucketName, bucketID, ownerUID, tj); err != nil {
|
||||
return r.fail(ctx, &b, "TagsFailed", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bucket default quota (applied to the owner).
|
||||
if q := b.Spec.Quota; q != nil {
|
||||
if err := r.Ceph.SetUserQuota(ctx, ownerUID, "bucket", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil {
|
||||
return r.fail(ctx, &b, "QuotaFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Render and apply the aggregate S3 policy from all BucketAccess grants.
|
||||
grants, principals, err := r.collectGrants(ctx, b.Namespace, b.Name)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "GrantsFailed", err)
|
||||
}
|
||||
policy, err := ceph.BuildBucketPolicy(bucketName, grants)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "PolicyBuildFailed", err)
|
||||
}
|
||||
if err := r.Ceph.SetBucketPolicy(ctx, bucketName, bucketID, ownerUID, policy); err != nil {
|
||||
return r.fail(ctx, &b, "PolicyFailed", err)
|
||||
}
|
||||
|
||||
b.Status.Phase = "Ready"
|
||||
b.Status.BucketName = bucketName
|
||||
b.Status.BucketID = bucketID
|
||||
b.Status.Owner = ownerUID
|
||||
b.Status.PolicyPrincipals = int32(principals)
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, true, "Provisioned", "bucket provisioned")
|
||||
if err := r.Status().Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
// collectGrants returns the deduplicated set of grants for a bucket, drawn from
|
||||
// every ready, non-deleting BucketAccess that references it, plus the count of
|
||||
// distinct principals.
|
||||
func (r *BucketReconciler) collectGrants(ctx context.Context, namespace, bucketRefName string) ([]ceph.Grant, int, error) {
|
||||
var list v1alpha1.BucketAccessList
|
||||
if err := r.List(ctx, &list, client.InNamespace(namespace)); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
principals := map[string]struct{}{}
|
||||
var grants []ceph.Grant
|
||||
for i := range list.Items {
|
||||
ba := &list.Items[i]
|
||||
if ba.Spec.BucketRef != bucketRefName {
|
||||
continue
|
||||
}
|
||||
if !ba.DeletionTimestamp.IsZero() {
|
||||
continue
|
||||
}
|
||||
if ba.Status.UID == "" {
|
||||
continue
|
||||
}
|
||||
key := ba.Status.UID + "|" + string(ba.Spec.Level)
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
principals[ba.Status.UID] = struct{}{}
|
||||
grants = append(grants, ceph.Grant{UID: ba.Status.UID, Level: string(ba.Spec.Level)})
|
||||
}
|
||||
return grants, len(principals), nil
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) pending(ctx context.Context, b *v1alpha1.Bucket, reason, msg string) (ctrl.Result, error) {
|
||||
b.Status.Phase = "Pending"
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, false, reason, msg)
|
||||
if err := r.Status().Update(ctx, b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) fail(ctx context.Context, b *v1alpha1.Bucket, reason string, cause error) (ctrl.Result, error) {
|
||||
b.Status.Phase = "Error"
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.Bucket{}).
|
||||
Watches(&v1alpha1.BucketAccess{}, handler.EnqueueRequestsFromMapFunc(r.bucketForAccess)).
|
||||
Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.bucketsForOwner)).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// bucketForAccess maps a BucketAccess change to its referenced Bucket.
|
||||
func (r *BucketReconciler) bucketForAccess(_ context.Context, obj client.Object) []reconcile.Request {
|
||||
ba, ok := obj.(*v1alpha1.BucketAccess)
|
||||
if !ok || ba.Spec.BucketRef == "" {
|
||||
return nil
|
||||
}
|
||||
return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}}}
|
||||
}
|
||||
|
||||
// bucketsForOwner maps an ObjectStoreUser change to every Bucket it owns.
|
||||
func (r *BucketReconciler) bucketsForOwner(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
osu, ok := obj.(*v1alpha1.ObjectStoreUser)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var list v1alpha1.BucketList
|
||||
if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil {
|
||||
return nil
|
||||
}
|
||||
var reqs []reconcile.Request
|
||||
for i := range list.Items {
|
||||
if list.Items[i].Spec.OwnerRef == osu.Name {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{
|
||||
Namespace: list.Items[i].Namespace, Name: list.Items[i].Name,
|
||||
}})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// BucketAccessReconciler ensures the principal for a grant exists (creating a
|
||||
// dedicated RGW user when none is referenced) and delivers its keys. The bucket
|
||||
// policy itself is owned and rendered by the Bucket controller, which watches
|
||||
// BucketAccess objects.
|
||||
type BucketAccessReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/finalizers,verbs=update
|
||||
|
||||
func (r *BucketAccessReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var ba v1alpha1.BucketAccess
|
||||
if err := r.Get(ctx, req.NamespacedName, &ba); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
managed := ba.Spec.UserRef == ""
|
||||
uid, err := r.resolveUID(ctx, &ba)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &ba, "ResolveFailed", err)
|
||||
}
|
||||
|
||||
if !ba.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&ba, finalizer) {
|
||||
// Only delete a user the operator created for this grant.
|
||||
if managed && uid != "" {
|
||||
if err := r.Ceph.DeleteUser(ctx, uid); err != nil {
|
||||
return r.fail(ctx, &ba, "DeleteFailed", err)
|
||||
}
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&ba, finalizer)
|
||||
if err := r.Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&ba, finalizer) {
|
||||
if err := r.Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the referenced Bucket so we can label the credential Secret and
|
||||
// gate the grant on the bucket existing.
|
||||
var bucket v1alpha1.Bucket
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}, &bucket); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.pending(ctx, &ba, "BucketMissing", fmt.Sprintf("waiting for Bucket %q", ba.Spec.BucketRef))
|
||||
}
|
||||
return r.fail(ctx, &ba, "BucketLookupFailed", err)
|
||||
}
|
||||
bucketName := orDefault(bucket.Status.BucketName, orDefault(bucket.Spec.BucketName, bucket.Name))
|
||||
|
||||
secretName := ba.Status.SecretName
|
||||
if managed {
|
||||
if uid == "" {
|
||||
uid = fmt.Sprintf("%s-%s", ba.Spec.BucketRef, ba.Name)
|
||||
}
|
||||
secretName = orDefault(ba.Spec.SecretName, ba.Name+"-rgw")
|
||||
|
||||
user, err := r.ensureUser(ctx, uid)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &ba, "UserFailed", err)
|
||||
}
|
||||
key, ok := user.S3Key()
|
||||
if !ok {
|
||||
return r.fail(ctx, &ba, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid))
|
||||
}
|
||||
if err := upsertSecret(ctx, r.Client, r.Scheme, &ba, secretName, ba.Namespace,
|
||||
credentialSecretData(key, uid, r.Endpoint, bucketName)); err != nil {
|
||||
return r.fail(ctx, &ba, "SecretFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
ba.Status.Phase = "Ready"
|
||||
ba.Status.UID = uid
|
||||
ba.Status.SecretName = secretName
|
||||
ba.Status.Bound = true
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, true, "Granted",
|
||||
fmt.Sprintf("%s access for %s applied to bucket %s", ba.Spec.Level, uid, bucketName))
|
||||
if err := r.Status().Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
logger.Info("bucket access reconciled", "bucket", bucketName, "uid", uid, "level", ba.Spec.Level)
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
// resolveUID returns the RGW uid this grant targets: the referenced
|
||||
// ObjectStoreUser's provisioned uid, or the managed uid derived from the spec.
|
||||
func (r *BucketAccessReconciler) resolveUID(ctx context.Context, ba *v1alpha1.BucketAccess) (string, error) {
|
||||
if ba.Spec.UserRef == "" {
|
||||
if ba.Spec.UID != "" {
|
||||
return ba.Spec.UID, nil
|
||||
}
|
||||
// Derived lazily in Reconcile once we know it is not a deletion no-op.
|
||||
return "", nil
|
||||
}
|
||||
var osu v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.UserRef}, &osu); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if osu.Status.UID == "" {
|
||||
return "", fmt.Errorf("ObjectStoreUser %q not ready", ba.Spec.UserRef)
|
||||
}
|
||||
return osu.Status.UID, nil
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) ensureUser(ctx context.Context, uid string) (*ceph.User, error) {
|
||||
if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) {
|
||||
if _, err := r.Ceph.CreateUser(ctx, ceph.UserSpec{UID: uid, DisplayName: uid}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Ceph.GetUser(ctx, uid)
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) pending(ctx context.Context, ba *v1alpha1.BucketAccess, reason, msg string) (ctrl.Result, error) {
|
||||
ba.Status.Phase = "Pending"
|
||||
ba.Status.Bound = false
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, false, reason, msg)
|
||||
if err := r.Status().Update(ctx, ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) fail(ctx context.Context, ba *v1alpha1.BucketAccess, reason string, cause error) (ctrl.Result, error) {
|
||||
ba.Status.Phase = "Error"
|
||||
ba.Status.Bound = false
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.BucketAccess{}).
|
||||
Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.accessForUser)).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// accessForUser maps an ObjectStoreUser change to every BucketAccess that
|
||||
// references it, so a grant binds as soon as its user becomes ready.
|
||||
func (r *BucketAccessReconciler) accessForUser(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
osu, ok := obj.(*v1alpha1.ObjectStoreUser)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var list v1alpha1.BucketAccessList
|
||||
if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil {
|
||||
return nil
|
||||
}
|
||||
var reqs []reconcile.Request
|
||||
for i := range list.Items {
|
||||
if list.Items[i].Spec.UserRef == osu.Name {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{
|
||||
Namespace: list.Items[i].Namespace, Name: list.Items[i].Name,
|
||||
}})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// finalizer guards external RGW state (users, buckets, policy statements) so it
|
||||
// is cleaned up before the Kubernetes object disappears.
|
||||
const finalizer = "ceph.unkin.net/finalizer"
|
||||
|
||||
// requeueSteady is the resync interval for healthy objects; it lets the
|
||||
// operator heal drift made directly against RGW.
|
||||
const requeueSteady = 10 * time.Minute
|
||||
|
||||
// requeueShort backs off on transient "waiting for a dependency" states.
|
||||
const requeueShort = 30 * time.Second
|
||||
|
||||
// setReady sets the standard Ready condition on a status conditions slice.
|
||||
func setReady(conds *[]metav1.Condition, gen int64, ok bool, reason, msg string) {
|
||||
status := metav1.ConditionFalse
|
||||
if ok {
|
||||
status = metav1.ConditionTrue
|
||||
}
|
||||
meta.SetStatusCondition(conds, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: status,
|
||||
ObservedGeneration: gen,
|
||||
Reason: reason,
|
||||
Message: truncate(msg, 32000),
|
||||
})
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func orDefault(v, def string) string {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// upsertSecret creates or updates an owner-referenced Opaque Secret with data.
|
||||
func upsertSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, name, namespace string, data map[string][]byte) error {
|
||||
sec := &corev1.Secret{}
|
||||
sec.Name = name
|
||||
sec.Namespace = namespace
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, c, sec, func() error {
|
||||
sec.Type = corev1.SecretTypeOpaque
|
||||
if sec.Data == nil {
|
||||
sec.Data = map[string][]byte{}
|
||||
}
|
||||
for k, v := range data {
|
||||
sec.Data[k] = v
|
||||
}
|
||||
return controllerutil.SetControllerReference(owner, sec, scheme)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// credentialSecretData assembles the conventional S3/AWS credential keys.
|
||||
func credentialSecretData(key ceph.UserKey, uid, endpoint, bucket string) map[string][]byte {
|
||||
data := map[string][]byte{
|
||||
"AWS_ACCESS_KEY_ID": []byte(key.AccessKey),
|
||||
"AWS_SECRET_ACCESS_KEY": []byte(key.SecretKey),
|
||||
"RGW_UID": []byte(uid),
|
||||
}
|
||||
if endpoint != "" {
|
||||
data["S3_ENDPOINT"] = []byte(endpoint)
|
||||
if host := hostOf(endpoint); host != "" {
|
||||
data["BUCKET_HOST"] = []byte(host)
|
||||
}
|
||||
}
|
||||
if bucket != "" {
|
||||
data["BUCKET_NAME"] = []byte(bucket)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func hostOf(endpoint string) string {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil || u.Host == "" {
|
||||
return endpoint
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// ObjectStoreUserReconciler provisions RGW users and delivers their keys.
|
||||
type ObjectStoreUserReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
|
||||
|
||||
func (r *ObjectStoreUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var osu v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, req.NamespacedName, &osu); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
uid := orDefault(osu.Spec.UID, osu.Name)
|
||||
secretName := orDefault(osu.Spec.SecretName, osu.Name+"-rgw")
|
||||
|
||||
if !osu.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&osu, finalizer) {
|
||||
if err := r.Ceph.DeleteUser(ctx, uid); err != nil {
|
||||
return r.fail(ctx, &osu, "DeleteFailed", err)
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&osu, finalizer)
|
||||
if err := r.Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&osu, finalizer) {
|
||||
if err := r.Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
spec := ceph.UserSpec{
|
||||
UID: uid,
|
||||
DisplayName: osu.Spec.DisplayName,
|
||||
Email: osu.Spec.Email,
|
||||
MaxBuckets: osu.Spec.MaxBuckets,
|
||||
Suspended: osu.Spec.Suspended,
|
||||
}
|
||||
|
||||
if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) {
|
||||
if _, err := r.Ceph.CreateUser(ctx, spec); err != nil {
|
||||
return r.fail(ctx, &osu, "CreateFailed", err)
|
||||
}
|
||||
logger.Info("created RGW user", "uid", uid)
|
||||
} else if err != nil {
|
||||
return r.fail(ctx, &osu, "LookupFailed", err)
|
||||
} else {
|
||||
if _, err := r.Ceph.UpdateUser(ctx, spec); err != nil {
|
||||
return r.fail(ctx, &osu, "UpdateFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
if q := osu.Spec.Quota; q != nil {
|
||||
if err := r.Ceph.SetUserQuota(ctx, uid, "user", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil {
|
||||
return r.fail(ctx, &osu, "QuotaFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
user, err := r.Ceph.GetUser(ctx, uid)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &osu, "LookupFailed", err)
|
||||
}
|
||||
key, ok := user.S3Key()
|
||||
if !ok {
|
||||
return r.fail(ctx, &osu, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid))
|
||||
}
|
||||
|
||||
if err := upsertSecret(ctx, r.Client, r.Scheme, &osu, secretName, osu.Namespace,
|
||||
credentialSecretData(key, uid, r.Endpoint, "")); err != nil {
|
||||
return r.fail(ctx, &osu, "SecretFailed", err)
|
||||
}
|
||||
|
||||
osu.Status.Phase = "Ready"
|
||||
osu.Status.UID = uid
|
||||
osu.Status.SecretName = secretName
|
||||
osu.Status.ObservedGeneration = osu.Generation
|
||||
setReady(&osu.Status.Conditions, osu.Generation, true, "Provisioned", "RGW user provisioned")
|
||||
if err := r.Status().Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
func (r *ObjectStoreUserReconciler) fail(ctx context.Context, osu *v1alpha1.ObjectStoreUser, reason string, cause error) (ctrl.Result, error) {
|
||||
osu.Status.Phase = "Error"
|
||||
osu.Status.ObservedGeneration = osu.Generation
|
||||
setReady(&osu.Status.Conditions, osu.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *ObjectStoreUserReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.ObjectStoreUser{}).
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// SetupAll registers every controller with the manager.
|
||||
func SetupAll(mgr ctrl.Manager, cephClient *ceph.Client, endpoint string) error {
|
||||
if err := (&ObjectStoreUserReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := (&BucketReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := (&BucketAccessReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user