commit 1ea1713d6eb7948d9ff39a068f29b1b6cf11ad9c Author: benvin Date: Sat Jul 18 00:07:22 2026 +1000 Initial cephrgw-operator: Ceph RGW buckets & keys via dashboard API Adds a Kubernetes operator that provisions Ceph RGW (S3) buckets and access keys declaratively through the Ceph manager dashboard REST API. Three CRDs in group ceph.unkin.net/v1alpha1: - ObjectStoreUser: creates an RGW user, delivers its key pair to a Secret - Bucket: creates an S3 bucket owned by an ObjectStoreUser; owns the bucket's aggregate S3 policy (union of all BucketAccess grants) - BucketAccess: grants read-only/read-write/full access, provisioning a dedicated user (or reusing a referenced one) and delivering RW/RO keys The internal/ceph client wraps the dashboard /api/auth, /api/rgw/user and /api/rgw/bucket endpoints with lazy token auth and re-auth on 401. Bucket policies are rendered deterministically and applied via the bucket policy API (Reef 18.2+). Credentials come from the cephrgw-credentials Secret via env. Includes generated CRDs/RBAC, samples, kind manifests, Woodpecker CI, and docs/ceph-setup.md covering the required Ceph dashboard account, RGW wiring and permissions. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..467f51d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/bin/ +*.out +*.test +.env diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml new file mode 100644 index 0000000..3d7c5f7 --- /dev/null +++ b/.woodpecker/build.yaml @@ -0,0 +1,10 @@ +when: + - event: pull_request + +steps: + - name: docker-build-operator + image: woodpeckerci/plugin-docker-buildx + settings: + repo: git.unkin.net/unkin/cephrgw-operator + dockerfile: Dockerfile.operator + dry_run: true diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml new file mode 100644 index 0000000..00fbdb3 --- /dev/null +++ b/.woodpecker/docker.yaml @@ -0,0 +1,17 @@ +when: + - event: tag + ref: refs/tags/v* + +steps: + - name: docker-operator + image: woodpeckerci/plugin-docker-buildx + settings: + registry: git.unkin.net + repo: git.unkin.net/unkin/cephrgw-operator + dockerfile: Dockerfile.operator + username: droneci + password: + from_secret: DRONECI_PASSWORD + tags: + - ${CI_COMMIT_TAG} + - latest diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml new file mode 100644 index 0000000..2dd88b8 --- /dev/null +++ b/.woodpecker/pre-commit.yaml @@ -0,0 +1,9 @@ +when: + - event: pull_request + +steps: + - name: pre-commit + image: golang:1.25 + commands: + - test -z "$(gofmt -l .)" + - go vet ./... diff --git a/.woodpecker/test.yaml b/.woodpecker/test.yaml new file mode 100644 index 0000000..75fc514 --- /dev/null +++ b/.woodpecker/test.yaml @@ -0,0 +1,8 @@ +when: + - event: pull_request + +steps: + - name: test + image: golang:1.25 + commands: + - go test -race -count=1 ./api/... ./internal/... diff --git a/Dockerfile.operator b/Dockerfile.operator new file mode 100644 index 0000000..9cb00a1 --- /dev/null +++ b/Dockerfile.operator @@ -0,0 +1,18 @@ +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /build + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o cephrgw-operator ./cmd/operator + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /build/cephrgw-operator /usr/local/bin/cephrgw-operator + +ENTRYPOINT ["cephrgw-operator"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d7accdd --- /dev/null +++ b/Makefile @@ -0,0 +1,57 @@ +.PHONY: build test lint fmt generate manifests docker-operator clean tidy patch minor major + +BINARY_OP := bin/cephrgw-operator +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev") + +build: tidy + go build -ldflags="-s -w" -o $(BINARY_OP) ./cmd/operator + +test: + go test -race -count=1 ./api/... ./internal/... + +lint: + go vet ./... + +fmt: + gofmt -w . + +CRD_BUNDLE := config/crd/install.yaml + +## generate: regenerate deepcopy, CRDs and RBAC from kubebuilder markers, then +## bundle every CRD into a single applyable manifest ($(CRD_BUNDLE)) so it can +## be referenced by a stable raw URL. +generate: + controller-gen object paths="./api/..." + controller-gen crd paths="./api/..." output:crd:artifacts:config=config/crd/bases + controller-gen rbac:roleName=cephrgw-operator paths="./internal/controller/..." output:rbac:dir=config/rbac + printf '# Generated by "make generate". DO NOT EDIT.\n' > $(CRD_BUNDLE) + cat config/crd/bases/*.yaml >> $(CRD_BUNDLE) + +manifests: generate + +docker-operator: + docker build -t cephrgw-operator:$(VERSION) -f Dockerfile.operator . + +clean: + rm -rf bin/ + +tidy: + go mod tidy + +_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1) +_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0) +_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1) +_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2) +_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3) + +patch: + @NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \ + git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW + +minor: + @NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \ + git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW + +major: + @NEW=v$(shell expr $(_MAJ) + 1).0.0; \ + git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc338c9 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +# cephrgw-operator + +A Kubernetes operator that provisions Ceph RGW (S3) **buckets** and **access +keys** declaratively, driving the Ceph **manager dashboard REST API**. You +describe a bucket, its owner, and who may read or write it as custom resources; +the operator creates the RGW users and bucket, delivers the access/secret keys +into Kubernetes Secrets, and maintains the bucket's S3 policy. + +It talks only to the dashboard API (e.g. `https://dashboard.ceph.unkin.net`) — +no RADOS access, no admin socket, no in-cluster Ceph required. + +## Custom resources + +| Kind | Short | Purpose | +|------|-------|---------| +| `ObjectStoreUser` | `osu` | An RGW S3 user. The operator creates it and writes its key pair into a Secret. | +| `Bucket` | `bkt` | An S3 bucket owned by an `ObjectStoreUser`. Owns the bucket's aggregate S3 policy. | +| `BucketAccess` | `ba` | Grants a user `read-only`, `read-write` or `full` access to a `Bucket`, delivering RW/RO keys. | + +### How access levels work + +The bucket **owner** (`Bucket.spec.ownerRef`) always has full control. Each +`BucketAccess` adds a principal to the bucket's S3 policy: + +- `read-only` → `s3:GetObject`, `s3:ListBucket` and friends. +- `read-write` → read plus `s3:PutObject` / `s3:DeleteObject` / multipart. +- `full` → `s3:*` on the bucket and its objects. + +If a `BucketAccess` omits `userRef`, the operator provisions a **dedicated** RGW +user for that grant and writes its keys into `spec.secretName` (default +`-rgw`). If `userRef` names an existing `ObjectStoreUser`, that user's own +credential Secret is reused and only the policy is extended. + +The `Bucket` controller renders the policy as the **union of every ready +`BucketAccess`** that targets it, so the result is convergent regardless of the +order objects are created or deleted. It watches `BucketAccess` and +`ObjectStoreUser`, re-reconciling the bucket whenever a grant or user changes. + +``` +ObjectStoreUser ──create user──▶ dashboard /api/rgw/user ──▶ Secret (AK/SK) +Bucket ──create bucket─▶ dashboard /api/rgw/bucket ─▶ owns S3 policy +BucketAccess ──ensure user───▶ dashboard /api/rgw/user ──▶ Secret (AK/SK, RW or RO) + └────── enqueues Bucket ──▶ PUT bucket_policy (aggregate) +``` + +## Credential Secrets + +Every credential Secret carries the conventional keys, ready to mount straight +into a workload: + +- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` +- `RGW_UID` +- `S3_ENDPOINT`, `BUCKET_HOST` (when `CEPH_RGW_ENDPOINT` is configured) +- `BUCKET_NAME` (on `BucketAccess` Secrets) + +Secrets are owner-referenced by the resource that produced them, so they are +garbage-collected when the resource is deleted. + +## Prerequisites + +The operator needs a dashboard login with the `rgw-manager` role, a dashboard +that is wired to RGW, and (for `read-only`/non-owner `read-write` grants) Ceph +**Reef 18.2+ / Squid**. See **[docs/ceph-setup.md](docs/ceph-setup.md)** for the +exact commands and the `cephrgw-credentials` Secret schema. + +## Quickstart + +```sh +kubectl apply -f config/samples/00-owner-user.yaml +kubectl apply -f config/samples/01-bucket.yaml +kubectl apply -f config/samples/02-access-readonly.yaml +kubectl apply -f config/samples/03-access-readwrite.yaml + +kubectl get osu,bkt,ba +kubectl get secret app-data-ro-rgw -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d +``` + +## Development + +```sh +make generate # regenerate deepcopy, CRDs and RBAC from kubebuilder markers +make build # build the operator binary +make test # go test -race +make lint fmt # go vet / gofmt +``` + +### Local (kind) + +```sh +kind create cluster --name cephrgw +docker build -t cephrgw-operator:dev -f Dockerfile.operator . +kind load docker-image cephrgw-operator:dev --name cephrgw + +kubectl apply -f config/crd/bases/ +kubectl apply -f hack/kind/manifests/ # edit the Secret first +``` + +## CI + +Woodpecker runs `pre-commit` (gofmt + vet), `test`, and a dry-run image `build` +on pull requests; pushing a `v*` tag builds and pushes +`git.unkin.net/unkin/cephrgw-operator` to the Gitea registry. Bump a release +with `make patch|minor|major`. + +## Notes & caveats + +- **Policy clearing.** Removing the last `BucketAccess` asks the dashboard to + clear the bucket policy. Not every release honours an empty policy string; if + a stale policy lingers, clear it once by hand. Adding/replacing grants always + works. +- **Per-bucket quota.** `Bucket.spec.quota` is applied as the owner's default + bucket quota via the dashboard, which is per-owner rather than strictly + per-bucket. Use distinct owners if you need independent bucket quotas. +- **Immutability.** `bucketName`, an `ObjectStoreUser`'s `uid`, and object lock + are fixed at creation; changing them on an existing object has no effect. diff --git a/api/v1alpha1/bucket_types.go b/api/v1alpha1/bucket_types.go new file mode 100644 index 0000000..d70401e --- /dev/null +++ b/api/v1alpha1/bucket_types.go @@ -0,0 +1,133 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ObjectLockMode is the S3 object-lock retention mode. +// +kubebuilder:validation:Enum=GOVERNANCE;COMPLIANCE +type ObjectLockMode string + +const ( + ObjectLockGovernance ObjectLockMode = "GOVERNANCE" + ObjectLockCompliance ObjectLockMode = "COMPLIANCE" +) + +// ObjectLock configures S3 object lock (WORM) on a bucket. Object lock can only +// be enabled at creation time and requires versioning. +type ObjectLock struct { + // Enabled turns on object lock for the bucket. + Enabled bool `json:"enabled"` + // Mode is the default retention mode applied to new objects. + // +optional + Mode ObjectLockMode `json:"mode,omitempty"` + // Days is the default retention period in days. Mutually exclusive with Years. + // +optional + Days *int32 `json:"days,omitempty"` + // Years is the default retention period in years. Mutually exclusive with Days. + // +optional + Years *int32 `json:"years,omitempty"` +} + +// BucketSpec defines a Ceph RGW (S3) bucket owned by an ObjectStoreUser. +type BucketSpec struct { + // BucketName is the S3 bucket name. Defaults to metadata.name. Immutable. + // +optional + BucketName string `json:"bucketName,omitempty"` + + // OwnerRef names the ObjectStoreUser (in this namespace) that owns the + // bucket. The owner always has full control; grant additional principals + // with BucketAccess objects. + OwnerRef string `json:"ownerRef"` + + // Zonegroup optionally pins the bucket to a specific RGW zonegroup. + // +optional + Zonegroup string `json:"zonegroup,omitempty"` + + // PlacementTarget optionally selects a non-default placement target/pool. + // +optional + PlacementTarget string `json:"placementTarget,omitempty"` + + // Versioning enables S3 object versioning on the bucket. + // +optional + Versioning bool `json:"versioning,omitempty"` + + // ObjectLock configures S3 object lock. Enabling it forces versioning on. + // +optional + ObjectLock *ObjectLock `json:"objectLock,omitempty"` + + // Quota optionally applies a bucket-level quota. + // +optional + Quota *Quota `json:"quota,omitempty"` + + // Tags are bucket tags (key/value) applied to the bucket. + // +optional + Tags map[string]string `json:"tags,omitempty"` + + // RetainOnDelete keeps the RGW bucket (and its objects) when the Bucket + // resource is deleted. By default the operator removes the empty bucket; + // it never purges objects unless PurgeOnDelete is also set. + // +optional + RetainOnDelete bool `json:"retainOnDelete,omitempty"` + + // PurgeOnDelete deletes the bucket together with all objects it contains + // when the Bucket resource is removed. Dangerous; defaults to false. + // +optional + PurgeOnDelete bool `json:"purgeOnDelete,omitempty"` +} + +// BucketStatus reports observed bucket state. +type BucketStatus struct { + // Phase is a coarse lifecycle summary (Pending/Ready/Error). + // +optional + Phase string `json:"phase,omitempty"` + // BucketName is the provisioned S3 bucket name. + // +optional + BucketName string `json:"bucketName,omitempty"` + // BucketID is the RGW internal bucket instance id. + // +optional + BucketID string `json:"bucketID,omitempty"` + // Owner is the RGW uid that owns the bucket. + // +optional + Owner string `json:"owner,omitempty"` + // PolicyPrincipals is the number of extra principals granted via + // BucketAccess and reflected in the bucket policy. + // +optional + PolicyPrincipals int32 `json:"policyPrincipals,omitempty"` + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=bkt +// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.status.bucketName` +// +kubebuilder:printcolumn:name="Owner",type=string,JSONPath=`.status.owner` +// +kubebuilder:printcolumn:name="Grants",type=integer,JSONPath=`.status.policyPrincipals` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` + +// Bucket is a Ceph RGW S3 bucket. +type Bucket struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BucketSpec `json:"spec,omitempty"` + Status BucketStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// BucketList contains a list of Bucket. +type BucketList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Bucket `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Bucket{}, &BucketList{}) +} diff --git a/api/v1alpha1/bucketaccess_types.go b/api/v1alpha1/bucketaccess_types.go new file mode 100644 index 0000000..399a4c8 --- /dev/null +++ b/api/v1alpha1/bucketaccess_types.go @@ -0,0 +1,101 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AccessLevel is the level of access a principal is granted to a bucket. The +// operator translates it into an S3 bucket policy statement. +// +kubebuilder:validation:Enum=read-only;read-write;full +type AccessLevel string + +const ( + // AccessReadOnly grants object GET and bucket LIST (s3:GetObject, + // s3:ListBucket and friends). + AccessReadOnly AccessLevel = "read-only" + // AccessReadWrite grants read plus object PUT/DELETE and multipart. + AccessReadWrite AccessLevel = "read-write" + // AccessFull grants s3:* on the bucket and its objects. + AccessFull AccessLevel = "full" +) + +// BucketAccessSpec grants an RGW user a level of access to a Bucket by +// maintaining a statement in the bucket's S3 policy. If UserRef is empty the +// operator provisions a dedicated user for this grant and writes its keys into +// a Secret; otherwise it grants an existing ObjectStoreUser. +type BucketAccessSpec struct { + // BucketRef names the Bucket (in this namespace) to grant access to. + BucketRef string `json:"bucketRef"` + + // Level is the access level to grant. + Level AccessLevel `json:"level"` + + // UserRef optionally names an existing ObjectStoreUser (in this namespace) + // to grant. When set, the operator does not create or delete a user and + // SecretName is ignored (that user already owns its own credential Secret). + // +optional + UserRef string `json:"userRef,omitempty"` + + // UID overrides the id of the dedicated user created when UserRef is empty. + // Defaults to "-". Ignored when UserRef is set. + // +optional + UID string `json:"uid,omitempty"` + + // SecretName is the Secret the operator writes credentials into for the + // dedicated user it creates (UserRef empty). Defaults to "-rgw". + // +optional + SecretName string `json:"secretName,omitempty"` +} + +// BucketAccessStatus reports observed grant state. +type BucketAccessStatus struct { + // Phase is a coarse lifecycle summary (Pending/Ready/Error). + // +optional + Phase string `json:"phase,omitempty"` + // UID is the RGW user id that was granted access. + // +optional + UID string `json:"uid,omitempty"` + // SecretName is the Secret holding the dedicated user's credentials, if any. + // +optional + SecretName string `json:"secretName,omitempty"` + // Bound reports whether the grant is reflected in the bucket policy. + // +optional + Bound bool `json:"bound,omitempty"` + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=ba +// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.spec.bucketRef` +// +kubebuilder:printcolumn:name="Level",type=string,JSONPath=`.spec.level` +// +kubebuilder:printcolumn:name="UID",type=string,JSONPath=`.status.uid` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` + +// BucketAccess grants an RGW user read-only, read-write or full access to a +// Bucket via the bucket's S3 policy. +type BucketAccess struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BucketAccessSpec `json:"spec,omitempty"` + Status BucketAccessStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// BucketAccessList contains a list of BucketAccess. +type BucketAccessList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BucketAccess `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BucketAccess{}, &BucketAccessList{}) +} diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go new file mode 100644 index 0000000..ed9c350 --- /dev/null +++ b/api/v1alpha1/common.go @@ -0,0 +1,27 @@ +package v1alpha1 + +// Quota describes an RGW quota applied to a user or a bucket. A nil limit (or a +// negative value) means unlimited for that dimension. +type Quota struct { + // Enabled turns the quota on. When false the other fields are ignored and + // the quota is disabled on the target. + // +kubebuilder:default=true + // +optional + Enabled bool `json:"enabled,omitempty"` + + // MaxSizeBytes caps the total size in bytes. Nil or negative means unlimited. + // +optional + MaxSizeBytes *int64 `json:"maxSizeBytes,omitempty"` + + // MaxObjects caps the number of objects. Nil or negative means unlimited. + // +optional + MaxObjects *int64 `json:"maxObjects,omitempty"` +} + +// SecretKeyRef points at a single key within a Secret in the same namespace. +type SecretKeyRef struct { + // Name is the Secret name. + Name string `json:"name"` + // Key is the data key within the Secret. + Key string `json:"key"` +} diff --git a/api/v1alpha1/doc.go b/api/v1alpha1/doc.go new file mode 100644 index 0000000..cfca2dc --- /dev/null +++ b/api/v1alpha1/doc.go @@ -0,0 +1,3 @@ +// +kubebuilder:object:generate=true +// +groupName=ceph.unkin.net +package v1alpha1 diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..13d4160 --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,17 @@ +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is the group/version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "ceph.unkin.net", Version: "v1alpha1"} + + // SchemeBuilder registers the API types with a runtime scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/objectstoreuser_types.go b/api/v1alpha1/objectstoreuser_types.go new file mode 100644 index 0000000..5439623 --- /dev/null +++ b/api/v1alpha1/objectstoreuser_types.go @@ -0,0 +1,91 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ObjectStoreUserSpec defines a Ceph RGW (S3) user. The operator creates the +// user through the Ceph dashboard API and writes its generated access/secret +// key pair into a Kubernetes Secret. The key material is never stored on the +// resource itself. +type ObjectStoreUserSpec struct { + // UID is the RGW user id. Defaults to metadata.name. Immutable once created. + // +optional + UID string `json:"uid,omitempty"` + + // DisplayName is the human-readable name for the user. Defaults to the UID. + // +optional + DisplayName string `json:"displayName,omitempty"` + + // Email is an optional email address recorded on the user. + // +optional + Email string `json:"email,omitempty"` + + // MaxBuckets caps how many buckets the user may own. A negative value + // disables bucket creation; 0 leaves the RGW default. Defaults to 1000. + // +kubebuilder:default=1000 + // +optional + MaxBuckets *int32 `json:"maxBuckets,omitempty"` + + // Suspended, when true, suspends the user so its keys stop working. + // +optional + Suspended bool `json:"suspended,omitempty"` + + // Quota optionally applies a user-level quota. + // +optional + Quota *Quota `json:"quota,omitempty"` + + // SecretName is the Secret the operator writes the access/secret key into. + // Defaults to "-rgw". The Secret holds AWS_ACCESS_KEY_ID, + // AWS_SECRET_ACCESS_KEY, BUCKET_HOST and the RGW uid. + // +optional + SecretName string `json:"secretName,omitempty"` +} + +// ObjectStoreUserStatus reports observed user state. +type ObjectStoreUserStatus struct { + // Phase is a coarse lifecycle summary (Pending/Ready/Error). + // +optional + Phase string `json:"phase,omitempty"` + // UID is the RGW user id that was provisioned. + // +optional + UID string `json:"uid,omitempty"` + // SecretName is the Secret holding the user's credentials. + // +optional + SecretName string `json:"secretName,omitempty"` + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=osu +// +kubebuilder:printcolumn:name="UID",type=string,JSONPath=`.status.uid` +// +kubebuilder:printcolumn:name="Secret",type=string,JSONPath=`.status.secretName` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` + +// ObjectStoreUser is a Ceph RGW S3 user whose keys are delivered into a Secret. +type ObjectStoreUser struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ObjectStoreUserSpec `json:"spec,omitempty"` + Status ObjectStoreUserStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ObjectStoreUserList contains a list of ObjectStoreUser. +type ObjectStoreUserList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ObjectStoreUser `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ObjectStoreUser{}, &ObjectStoreUserList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..ba1499f --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,390 @@ +//go:build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Bucket) DeepCopyInto(out *Bucket) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Bucket. +func (in *Bucket) DeepCopy() *Bucket { + if in == nil { + return nil + } + out := new(Bucket) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Bucket) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketAccess) DeepCopyInto(out *BucketAccess) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccess. +func (in *BucketAccess) DeepCopy() *BucketAccess { + if in == nil { + return nil + } + out := new(BucketAccess) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BucketAccess) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketAccessList) DeepCopyInto(out *BucketAccessList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BucketAccess, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessList. +func (in *BucketAccessList) DeepCopy() *BucketAccessList { + if in == nil { + return nil + } + out := new(BucketAccessList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BucketAccessList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketAccessSpec) DeepCopyInto(out *BucketAccessSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessSpec. +func (in *BucketAccessSpec) DeepCopy() *BucketAccessSpec { + if in == nil { + return nil + } + out := new(BucketAccessSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketAccessStatus) DeepCopyInto(out *BucketAccessStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessStatus. +func (in *BucketAccessStatus) DeepCopy() *BucketAccessStatus { + if in == nil { + return nil + } + out := new(BucketAccessStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketList) DeepCopyInto(out *BucketList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Bucket, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketList. +func (in *BucketList) DeepCopy() *BucketList { + if in == nil { + return nil + } + out := new(BucketList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BucketList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketSpec) DeepCopyInto(out *BucketSpec) { + *out = *in + if in.ObjectLock != nil { + in, out := &in.ObjectLock, &out.ObjectLock + *out = new(ObjectLock) + (*in).DeepCopyInto(*out) + } + if in.Quota != nil { + in, out := &in.Quota, &out.Quota + *out = new(Quota) + (*in).DeepCopyInto(*out) + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketSpec. +func (in *BucketSpec) DeepCopy() *BucketSpec { + if in == nil { + return nil + } + out := new(BucketSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketStatus) DeepCopyInto(out *BucketStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus. +func (in *BucketStatus) DeepCopy() *BucketStatus { + if in == nil { + return nil + } + out := new(BucketStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectLock) DeepCopyInto(out *ObjectLock) { + *out = *in + if in.Days != nil { + in, out := &in.Days, &out.Days + *out = new(int32) + **out = **in + } + if in.Years != nil { + in, out := &in.Years, &out.Years + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectLock. +func (in *ObjectLock) DeepCopy() *ObjectLock { + if in == nil { + return nil + } + out := new(ObjectLock) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreUser) DeepCopyInto(out *ObjectStoreUser) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUser. +func (in *ObjectStoreUser) DeepCopy() *ObjectStoreUser { + if in == nil { + return nil + } + out := new(ObjectStoreUser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ObjectStoreUser) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreUserList) DeepCopyInto(out *ObjectStoreUserList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ObjectStoreUser, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserList. +func (in *ObjectStoreUserList) DeepCopy() *ObjectStoreUserList { + if in == nil { + return nil + } + out := new(ObjectStoreUserList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ObjectStoreUserList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreUserSpec) DeepCopyInto(out *ObjectStoreUserSpec) { + *out = *in + if in.MaxBuckets != nil { + in, out := &in.MaxBuckets, &out.MaxBuckets + *out = new(int32) + **out = **in + } + if in.Quota != nil { + in, out := &in.Quota, &out.Quota + *out = new(Quota) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserSpec. +func (in *ObjectStoreUserSpec) DeepCopy() *ObjectStoreUserSpec { + if in == nil { + return nil + } + out := new(ObjectStoreUserSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreUserStatus) DeepCopyInto(out *ObjectStoreUserStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserStatus. +func (in *ObjectStoreUserStatus) DeepCopy() *ObjectStoreUserStatus { + if in == nil { + return nil + } + out := new(ObjectStoreUserStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Quota) DeepCopyInto(out *Quota) { + *out = *in + if in.MaxSizeBytes != nil { + in, out := &in.MaxSizeBytes, &out.MaxSizeBytes + *out = new(int64) + **out = **in + } + if in.MaxObjects != nil { + in, out := &in.MaxObjects, &out.MaxObjects + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Quota. +func (in *Quota) DeepCopy() *Quota { + if in == nil { + return nil + } + out := new(Quota) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyRef. +func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { + if in == nil { + return nil + } + out := new(SecretKeyRef) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/operator/main.go b/cmd/operator/main.go new file mode 100644 index 0000000..4ca6757 --- /dev/null +++ b/cmd/operator/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "flag" + "os" + "time" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + cephv1alpha1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1" + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" + "git.unkin.net/unkin/cephrgw-operator/internal/controller" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(cephv1alpha1.AddToScheme(scheme)) +} + +func main() { + var metricsAddr, probeAddr string + var leaderElect bool + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "metrics endpoint address") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "health probe address") + flag.BoolVar(&leaderElect, "leader-elect", false, "enable leader election") + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseDevMode(false))) + logger := ctrl.Log.WithName("setup") + + cephCfg, endpoint, err := cephConfigFromEnv() + if err != nil { + logger.Error(err, "invalid Ceph dashboard configuration") + os.Exit(1) + } + cephClient, err := ceph.NewClient(cephCfg) + if err != nil { + logger.Error(err, "unable to build Ceph dashboard client") + os.Exit(1) + } + // Fail fast on obviously-broken credentials, but do not block startup on a + // transiently unreachable dashboard. + pingCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + if err := cephClient.Ping(pingCtx); err != nil { + logger.Error(err, "initial dashboard authentication failed; continuing and will retry per-reconcile") + } + cancel() + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: metricsAddr}, + HealthProbeBindAddress: probeAddr, + LeaderElection: leaderElect, + LeaderElectionID: "cephrgw-operator", + }) + if err != nil { + logger.Error(err, "unable to create manager") + os.Exit(1) + } + + if err := controller.SetupAll(mgr, cephClient, endpoint); err != nil { + logger.Error(err, "unable to set up controllers") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + logger.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + logger.Error(err, "unable to set up ready check") + os.Exit(1) + } + + logger.Info("starting cephrgw-operator") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + logger.Error(err, "manager exited with error") + os.Exit(1) + } +} + +// cephConfigFromEnv reads dashboard connection settings from the environment, +// which the deployment sources from the cephrgw-credentials Secret. +func cephConfigFromEnv() (ceph.Config, string, error) { + cfg := ceph.Config{ + BaseURL: os.Getenv("CEPH_DASHBOARD_URL"), + Username: os.Getenv("CEPH_DASHBOARD_USERNAME"), + Password: os.Getenv("CEPH_DASHBOARD_PASSWORD"), + Insecure: os.Getenv("CEPH_DASHBOARD_INSECURE") == "true", + } + if f := os.Getenv("CEPH_DASHBOARD_CA_FILE"); f != "" { + b, err := os.ReadFile(f) + if err != nil { + return cfg, "", err + } + cfg.CACert = b + } else if inline := os.Getenv("CEPH_DASHBOARD_CA"); inline != "" { + cfg.CACert = []byte(inline) + } + endpoint := os.Getenv("CEPH_RGW_ENDPOINT") + return cfg, endpoint, nil +} diff --git a/config/crd/bases/ceph.unkin.net_bucketaccesses.yaml b/config/crd/bases/ceph.unkin.net_bucketaccesses.yaml new file mode 100644 index 0000000..e283c8e --- /dev/null +++ b/config/crd/bases/ceph.unkin.net_bucketaccesses.yaml @@ -0,0 +1,178 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: bucketaccesses.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: BucketAccess + listKind: BucketAccessList + plural: bucketaccesses + shortNames: + - ba + singular: bucketaccess + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.bucketRef + name: Bucket + type: string + - jsonPath: .spec.level + name: Level + type: string + - jsonPath: .status.uid + name: UID + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BucketAccess grants an RGW user read-only, read-write or full access to a + Bucket via the bucket's S3 policy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + BucketAccessSpec grants an RGW user a level of access to a Bucket by + maintaining a statement in the bucket's S3 policy. If UserRef is empty the + operator provisions a dedicated user for this grant and writes its keys into + a Secret; otherwise it grants an existing ObjectStoreUser. + properties: + bucketRef: + description: BucketRef names the Bucket (in this namespace) to grant + access to. + type: string + level: + description: Level is the access level to grant. + enum: + - read-only + - read-write + - full + type: string + secretName: + description: |- + SecretName is the Secret the operator writes credentials into for the + dedicated user it creates (UserRef empty). Defaults to "-rgw". + type: string + uid: + description: |- + UID overrides the id of the dedicated user created when UserRef is empty. + Defaults to "-". Ignored when UserRef is set. + type: string + userRef: + description: |- + UserRef optionally names an existing ObjectStoreUser (in this namespace) + to grant. When set, the operator does not create or delete a user and + SecretName is ignored (that user already owns its own credential Secret). + type: string + required: + - bucketRef + - level + type: object + status: + description: BucketAccessStatus reports observed grant state. + properties: + bound: + description: Bound reports whether the grant is reflected in the bucket + policy. + type: boolean + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + secretName: + description: SecretName is the Secret holding the dedicated user's + credentials, if any. + type: string + uid: + description: UID is the RGW user id that was granted access. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/ceph.unkin.net_buckets.yaml b/config/crd/bases/ceph.unkin.net_buckets.yaml new file mode 100644 index 0000000..db8f6b2 --- /dev/null +++ b/config/crd/bases/ceph.unkin.net_buckets.yaml @@ -0,0 +1,232 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: buckets.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: Bucket + listKind: BucketList + plural: buckets + shortNames: + - bkt + singular: bucket + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.bucketName + name: Bucket + type: string + - jsonPath: .status.owner + name: Owner + type: string + - jsonPath: .status.policyPrincipals + name: Grants + type: integer + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Bucket is a Ceph RGW S3 bucket. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BucketSpec defines a Ceph RGW (S3) bucket owned by an ObjectStoreUser. + properties: + bucketName: + description: BucketName is the S3 bucket name. Defaults to metadata.name. + Immutable. + type: string + objectLock: + description: ObjectLock configures S3 object lock. Enabling it forces + versioning on. + properties: + days: + description: Days is the default retention period in days. Mutually + exclusive with Years. + format: int32 + type: integer + enabled: + description: Enabled turns on object lock for the bucket. + type: boolean + mode: + description: Mode is the default retention mode applied to new + objects. + enum: + - GOVERNANCE + - COMPLIANCE + type: string + years: + description: Years is the default retention period in years. Mutually + exclusive with Days. + format: int32 + type: integer + required: + - enabled + type: object + ownerRef: + description: |- + OwnerRef names the ObjectStoreUser (in this namespace) that owns the + bucket. The owner always has full control; grant additional principals + with BucketAccess objects. + type: string + placementTarget: + description: PlacementTarget optionally selects a non-default placement + target/pool. + type: string + purgeOnDelete: + description: |- + PurgeOnDelete deletes the bucket together with all objects it contains + when the Bucket resource is removed. Dangerous; defaults to false. + type: boolean + quota: + description: Quota optionally applies a bucket-level quota. + properties: + enabled: + default: true + description: |- + Enabled turns the quota on. When false the other fields are ignored and + the quota is disabled on the target. + type: boolean + maxObjects: + description: MaxObjects caps the number of objects. Nil or negative + means unlimited. + format: int64 + type: integer + maxSizeBytes: + description: MaxSizeBytes caps the total size in bytes. Nil or + negative means unlimited. + format: int64 + type: integer + type: object + retainOnDelete: + description: |- + RetainOnDelete keeps the RGW bucket (and its objects) when the Bucket + resource is deleted. By default the operator removes the empty bucket; + it never purges objects unless PurgeOnDelete is also set. + type: boolean + tags: + additionalProperties: + type: string + description: Tags are bucket tags (key/value) applied to the bucket. + type: object + versioning: + description: Versioning enables S3 object versioning on the bucket. + type: boolean + zonegroup: + description: Zonegroup optionally pins the bucket to a specific RGW + zonegroup. + type: string + required: + - ownerRef + type: object + status: + description: BucketStatus reports observed bucket state. + properties: + bucketID: + description: BucketID is the RGW internal bucket instance id. + type: string + bucketName: + description: BucketName is the provisioned S3 bucket name. + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + owner: + description: Owner is the RGW uid that owns the bucket. + type: string + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + policyPrincipals: + description: |- + PolicyPrincipals is the number of extra principals granted via + BucketAccess and reflected in the bucket policy. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/ceph.unkin.net_objectstoreusers.yaml b/config/crd/bases/ceph.unkin.net_objectstoreusers.yaml new file mode 100644 index 0000000..a9ec44e --- /dev/null +++ b/config/crd/bases/ceph.unkin.net_objectstoreusers.yaml @@ -0,0 +1,187 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: objectstoreusers.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: ObjectStoreUser + listKind: ObjectStoreUserList + plural: objectstoreusers + shortNames: + - osu + singular: objectstoreuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.uid + name: UID + type: string + - jsonPath: .status.secretName + name: Secret + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ObjectStoreUser is a Ceph RGW S3 user whose keys are delivered + into a Secret. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ObjectStoreUserSpec defines a Ceph RGW (S3) user. The operator creates the + user through the Ceph dashboard API and writes its generated access/secret + key pair into a Kubernetes Secret. The key material is never stored on the + resource itself. + properties: + displayName: + description: DisplayName is the human-readable name for the user. + Defaults to the UID. + type: string + email: + description: Email is an optional email address recorded on the user. + type: string + maxBuckets: + default: 1000 + description: |- + MaxBuckets caps how many buckets the user may own. A negative value + disables bucket creation; 0 leaves the RGW default. Defaults to 1000. + format: int32 + type: integer + quota: + description: Quota optionally applies a user-level quota. + properties: + enabled: + default: true + description: |- + Enabled turns the quota on. When false the other fields are ignored and + the quota is disabled on the target. + type: boolean + maxObjects: + description: MaxObjects caps the number of objects. Nil or negative + means unlimited. + format: int64 + type: integer + maxSizeBytes: + description: MaxSizeBytes caps the total size in bytes. Nil or + negative means unlimited. + format: int64 + type: integer + type: object + secretName: + description: |- + SecretName is the Secret the operator writes the access/secret key into. + Defaults to "-rgw". The Secret holds AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, BUCKET_HOST and the RGW uid. + type: string + suspended: + description: Suspended, when true, suspends the user so its keys stop + working. + type: boolean + uid: + description: UID is the RGW user id. Defaults to metadata.name. Immutable + once created. + type: string + type: object + status: + description: ObjectStoreUserStatus reports observed user state. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + secretName: + description: SecretName is the Secret holding the user's credentials. + type: string + uid: + description: UID is the RGW user id that was provisioned. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/install.yaml b/config/crd/install.yaml new file mode 100644 index 0000000..64614b0 --- /dev/null +++ b/config/crd/install.yaml @@ -0,0 +1,598 @@ +# Generated by "make generate". DO NOT EDIT. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: bucketaccesses.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: BucketAccess + listKind: BucketAccessList + plural: bucketaccesses + shortNames: + - ba + singular: bucketaccess + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.bucketRef + name: Bucket + type: string + - jsonPath: .spec.level + name: Level + type: string + - jsonPath: .status.uid + name: UID + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BucketAccess grants an RGW user read-only, read-write or full access to a + Bucket via the bucket's S3 policy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + BucketAccessSpec grants an RGW user a level of access to a Bucket by + maintaining a statement in the bucket's S3 policy. If UserRef is empty the + operator provisions a dedicated user for this grant and writes its keys into + a Secret; otherwise it grants an existing ObjectStoreUser. + properties: + bucketRef: + description: BucketRef names the Bucket (in this namespace) to grant + access to. + type: string + level: + description: Level is the access level to grant. + enum: + - read-only + - read-write + - full + type: string + secretName: + description: |- + SecretName is the Secret the operator writes credentials into for the + dedicated user it creates (UserRef empty). Defaults to "-rgw". + type: string + uid: + description: |- + UID overrides the id of the dedicated user created when UserRef is empty. + Defaults to "-". Ignored when UserRef is set. + type: string + userRef: + description: |- + UserRef optionally names an existing ObjectStoreUser (in this namespace) + to grant. When set, the operator does not create or delete a user and + SecretName is ignored (that user already owns its own credential Secret). + type: string + required: + - bucketRef + - level + type: object + status: + description: BucketAccessStatus reports observed grant state. + properties: + bound: + description: Bound reports whether the grant is reflected in the bucket + policy. + type: boolean + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + secretName: + description: SecretName is the Secret holding the dedicated user's + credentials, if any. + type: string + uid: + description: UID is the RGW user id that was granted access. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: buckets.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: Bucket + listKind: BucketList + plural: buckets + shortNames: + - bkt + singular: bucket + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.bucketName + name: Bucket + type: string + - jsonPath: .status.owner + name: Owner + type: string + - jsonPath: .status.policyPrincipals + name: Grants + type: integer + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Bucket is a Ceph RGW S3 bucket. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BucketSpec defines a Ceph RGW (S3) bucket owned by an ObjectStoreUser. + properties: + bucketName: + description: BucketName is the S3 bucket name. Defaults to metadata.name. + Immutable. + type: string + objectLock: + description: ObjectLock configures S3 object lock. Enabling it forces + versioning on. + properties: + days: + description: Days is the default retention period in days. Mutually + exclusive with Years. + format: int32 + type: integer + enabled: + description: Enabled turns on object lock for the bucket. + type: boolean + mode: + description: Mode is the default retention mode applied to new + objects. + enum: + - GOVERNANCE + - COMPLIANCE + type: string + years: + description: Years is the default retention period in years. Mutually + exclusive with Days. + format: int32 + type: integer + required: + - enabled + type: object + ownerRef: + description: |- + OwnerRef names the ObjectStoreUser (in this namespace) that owns the + bucket. The owner always has full control; grant additional principals + with BucketAccess objects. + type: string + placementTarget: + description: PlacementTarget optionally selects a non-default placement + target/pool. + type: string + purgeOnDelete: + description: |- + PurgeOnDelete deletes the bucket together with all objects it contains + when the Bucket resource is removed. Dangerous; defaults to false. + type: boolean + quota: + description: Quota optionally applies a bucket-level quota. + properties: + enabled: + default: true + description: |- + Enabled turns the quota on. When false the other fields are ignored and + the quota is disabled on the target. + type: boolean + maxObjects: + description: MaxObjects caps the number of objects. Nil or negative + means unlimited. + format: int64 + type: integer + maxSizeBytes: + description: MaxSizeBytes caps the total size in bytes. Nil or + negative means unlimited. + format: int64 + type: integer + type: object + retainOnDelete: + description: |- + RetainOnDelete keeps the RGW bucket (and its objects) when the Bucket + resource is deleted. By default the operator removes the empty bucket; + it never purges objects unless PurgeOnDelete is also set. + type: boolean + tags: + additionalProperties: + type: string + description: Tags are bucket tags (key/value) applied to the bucket. + type: object + versioning: + description: Versioning enables S3 object versioning on the bucket. + type: boolean + zonegroup: + description: Zonegroup optionally pins the bucket to a specific RGW + zonegroup. + type: string + required: + - ownerRef + type: object + status: + description: BucketStatus reports observed bucket state. + properties: + bucketID: + description: BucketID is the RGW internal bucket instance id. + type: string + bucketName: + description: BucketName is the provisioned S3 bucket name. + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + owner: + description: Owner is the RGW uid that owns the bucket. + type: string + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + policyPrincipals: + description: |- + PolicyPrincipals is the number of extra principals granted via + BucketAccess and reflected in the bucket policy. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: objectstoreusers.ceph.unkin.net +spec: + group: ceph.unkin.net + names: + kind: ObjectStoreUser + listKind: ObjectStoreUserList + plural: objectstoreusers + shortNames: + - osu + singular: objectstoreuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.uid + name: UID + type: string + - jsonPath: .status.secretName + name: Secret + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ObjectStoreUser is a Ceph RGW S3 user whose keys are delivered + into a Secret. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ObjectStoreUserSpec defines a Ceph RGW (S3) user. The operator creates the + user through the Ceph dashboard API and writes its generated access/secret + key pair into a Kubernetes Secret. The key material is never stored on the + resource itself. + properties: + displayName: + description: DisplayName is the human-readable name for the user. + Defaults to the UID. + type: string + email: + description: Email is an optional email address recorded on the user. + type: string + maxBuckets: + default: 1000 + description: |- + MaxBuckets caps how many buckets the user may own. A negative value + disables bucket creation; 0 leaves the RGW default. Defaults to 1000. + format: int32 + type: integer + quota: + description: Quota optionally applies a user-level quota. + properties: + enabled: + default: true + description: |- + Enabled turns the quota on. When false the other fields are ignored and + the quota is disabled on the target. + type: boolean + maxObjects: + description: MaxObjects caps the number of objects. Nil or negative + means unlimited. + format: int64 + type: integer + maxSizeBytes: + description: MaxSizeBytes caps the total size in bytes. Nil or + negative means unlimited. + format: int64 + type: integer + type: object + secretName: + description: |- + SecretName is the Secret the operator writes the access/secret key into. + Defaults to "-rgw". The Secret holds AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, BUCKET_HOST and the RGW uid. + type: string + suspended: + description: Suspended, when true, suspends the user so its keys stop + working. + type: boolean + uid: + description: UID is the RGW user id. Defaults to metadata.name. Immutable + once created. + type: string + type: object + status: + description: ObjectStoreUserStatus reports observed user state. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + phase: + description: Phase is a coarse lifecycle summary (Pending/Ready/Error). + type: string + secretName: + description: SecretName is the Secret holding the user's credentials. + type: string + uid: + description: UID is the RGW user id that was provisioned. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..e705659 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephrgw-operator +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ceph.unkin.net + resources: + - bucketaccesses + - buckets + - objectstoreusers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ceph.unkin.net + resources: + - bucketaccesses/finalizers + - buckets/finalizers + - objectstoreusers/finalizers + verbs: + - update +- apiGroups: + - ceph.unkin.net + resources: + - bucketaccesses/status + - buckets/status + - objectstoreusers/status + verbs: + - get + - patch + - update diff --git a/config/samples/00-owner-user.yaml b/config/samples/00-owner-user.yaml new file mode 100644 index 0000000..68782ad --- /dev/null +++ b/config/samples/00-owner-user.yaml @@ -0,0 +1,13 @@ +# The owner of a bucket. The operator creates an RGW user and writes its +# access/secret key pair into the Secret "app-owner-rgw". +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: app-owner + namespace: default +spec: + displayName: "Application bucket owner" + maxBuckets: 50 + quota: + enabled: true + maxSizeBytes: 107374182400 # 100 GiB diff --git a/config/samples/01-bucket.yaml b/config/samples/01-bucket.yaml new file mode 100644 index 0000000..ff478ee --- /dev/null +++ b/config/samples/01-bucket.yaml @@ -0,0 +1,16 @@ +# A bucket owned by the app-owner user, with versioning enabled. +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: app-data + namespace: default +spec: + bucketName: app-data + ownerRef: app-owner + versioning: true + tags: + team: platform + env: prod + # By default the operator deletes the (empty) bucket when this object is + # removed. Set retainOnDelete: true to keep it, or purgeOnDelete: true to + # delete it together with all objects. diff --git a/config/samples/02-access-readonly.yaml b/config/samples/02-access-readonly.yaml new file mode 100644 index 0000000..f8e0a6a --- /dev/null +++ b/config/samples/02-access-readonly.yaml @@ -0,0 +1,13 @@ +# Read-only credentials for the bucket. Because no userRef is given, the +# operator provisions a dedicated RGW user for this grant and writes its keys +# into the Secret "app-data-ro-rgw". The Bucket's S3 policy is updated to grant +# this user GetObject/ListBucket only. +apiVersion: ceph.unkin.net/v1alpha1 +kind: BucketAccess +metadata: + name: app-data-ro + namespace: default +spec: + bucketRef: app-data + level: read-only + secretName: app-data-ro-rgw diff --git a/config/samples/03-access-readwrite.yaml b/config/samples/03-access-readwrite.yaml new file mode 100644 index 0000000..2c3effd --- /dev/null +++ b/config/samples/03-access-readwrite.yaml @@ -0,0 +1,34 @@ +# Read-write credentials for the bucket, delivered to a dedicated user and +# Secret "app-data-rw-rgw". The owner (app-owner) always retains full control; +# this grant is for a separate workload that needs to read and write objects +# but must not manage the bucket itself. +apiVersion: ceph.unkin.net/v1alpha1 +kind: BucketAccess +metadata: + name: app-data-rw + namespace: default +spec: + bucketRef: app-data + level: read-write + secretName: app-data-rw-rgw +--- +# Alternatively, grant an *existing* ObjectStoreUser access to the bucket by +# name. Here no user or Secret is created; the shared user's own credential +# Secret is used, and the bucket policy is extended to include it. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: analytics + namespace: default +spec: + displayName: "Analytics pipeline" +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: BucketAccess +metadata: + name: app-data-analytics + namespace: default +spec: + bucketRef: app-data + level: read-only + userRef: analytics diff --git a/docs/ceph-setup.md b/docs/ceph-setup.md new file mode 100644 index 0000000..2638c3c --- /dev/null +++ b/docs/ceph-setup.md @@ -0,0 +1,175 @@ +# Ceph setup: credentials and permissions the operator needs + +`cephrgw-operator` never talks to RADOS or the RGW admin socket directly. It +drives the **Ceph manager dashboard REST API** (the same API the web dashboard +uses) at `https://dashboard.ceph.unkin.net`. Everything below is about giving +the operator a dashboard login with enough RGW authority, and making sure the +dashboard itself is wired to your RGW. + +There are **two** credentials involved. Don't confuse them: + +| # | Credential | Who uses it | What it is | +|---|------------|-------------|------------| +| 1 | Dashboard login (username + password) | the operator → `POST /api/auth` | a **dashboard account** with the `rgw-manager` role | +| 2 | RGW admin connection | the dashboard → RGW | a **radosgw system user** (access/secret key) the dashboard is configured with | + +The operator only holds #1. #2 is what actually lets the dashboard create RGW +users, buckets and bucket policies on the operator's behalf, so it must exist +and be privileged. + +--- + +## 1. Create the dashboard login for the operator + +Create a dedicated dashboard user with the built-in **`rgw-manager`** role. That +role grants full create/read/update/delete on the dashboard's `rgw` scope +(users, buckets, policies) and nothing else — least privilege for this operator. + +```bash +# Put the password in a file so it never lands in shell history. +printf '%s' 'REPLACE-WITH-A-STRONG-PASSWORD' > /tmp/cephrgw.pw + +ceph dashboard ac-user-create k8s-cephrgw-operator -i /tmp/cephrgw.pw rgw-manager + +rm -f /tmp/cephrgw.pw +``` + +If your Ceph version wants the arguments in a different order, check +`ceph dashboard ac-user-create -h`. To confirm the role exists and what it +grants: + +```bash +ceph dashboard ac-role-show rgw-manager +``` + +> Prefer `rgw-manager` over `administrator`. The operator only needs RGW +> authority; giving it full dashboard admin is unnecessary blast radius. + +## 2. Make sure the dashboard can manage RGW + +The dashboard performs RGW operations through a **radosgw system user**. On +recent Ceph (Pacific and later) the mgr/dashboard module usually auto-discovers +and configures this. Verify it first: + +```bash +ceph dashboard get-rgw-api-access-key # should print a key, not empty +``` + +If it is empty, create a system user and point the dashboard at it: + +```bash +radosgw-admin user create \ + --uid=dashboard \ + --display-name="Ceph Dashboard" \ + --system + +# Feed the returned keys to the dashboard. +radosgw-admin user info --uid=dashboard \ + | jq -r '.keys[0].access_key' > /tmp/ak +radosgw-admin user info --uid=dashboard \ + | jq -r '.keys[0].secret_key' > /tmp/sk + +ceph dashboard set-rgw-api-access-key -i /tmp/ak +ceph dashboard set-rgw-api-secret-key -i /tmp/sk +rm -f /tmp/ak /tmp/sk +``` + +A `--system` user has the admin caps the dashboard needs to create/delete RGW +users and buckets and to set bucket policies on any bucket. If you would rather +not use `--system`, grant an equivalent admin cap set instead: + +```bash +radosgw-admin caps add --uid=dashboard \ + --caps="users=*;buckets=*;metadata=*;usage=read;zone=read" +``` + +If the dashboard reaches RGW over TLS with a private CA, you may also need: + +```bash +ceph dashboard set-rgw-api-ssl-verify true # keep verification on in prod +``` + +## 3. Bucket policy support (read-only / non-owner read-write) + +The operator enforces `read-only` and non-owner `read-write` grants by writing +an **S3 bucket policy** through the dashboard's bucket API (the `bucket_policy` +field on `PUT /api/rgw/bucket/{name}`). That field is available on **Ceph Reef +18.2+ / Squid**. On older releases bucket creation and owner (`full`) access +still work, but policy-based grants will fail — upgrade the cluster, or only use +owner credentials, if you are pre-Reef. + +Check your version: + +```bash +ceph versions | jq -r '.mon | keys[]' +``` + +## 4. (Optional) S3 endpoint for consumers + +The operator can stamp the S3 endpoint into every credential Secret it writes +(`S3_ENDPOINT` and `BUCKET_HOST`) so applications don't have to hard-code it. +This is the RGW/S3 endpoint your clients use — **not** the dashboard URL. Provide +it via `CEPH_RGW_ENDPOINT` (see below); if unset, those keys are simply omitted. + +--- + +## 5. Give the operator its credentials (the `cephrgw-credentials` Secret) + +The operator reads its configuration from environment variables, which the +deployment sources from a Secret named **`cephrgw-credentials`** in its namespace +(`cephrgw-system`). The Secret data keys map 1:1 to the env vars: + +| Secret key | Required | Meaning | +|------------|----------|---------| +| `CEPH_DASHBOARD_URL` | yes | dashboard base URL, e.g. `https://dashboard.ceph.unkin.net` | +| `CEPH_DASHBOARD_USERNAME` | yes | the `rgw-manager` account from step 1 | +| `CEPH_DASHBOARD_PASSWORD` | yes | its password | +| `CEPH_RGW_ENDPOINT` | no | S3 endpoint written into consumer Secrets | +| `CEPH_DASHBOARD_CA` | no | PEM CA bundle to verify the dashboard TLS cert (inline) | +| `CEPH_DASHBOARD_CA_FILE` | no | path to a mounted CA file (alternative to the above) | +| `CEPH_DASHBOARD_INSECURE` | no | `"true"` to skip TLS verification (dev only) | + +Create it directly: + +```bash +kubectl -n cephrgw-system create secret generic cephrgw-credentials \ + --from-literal=CEPH_DASHBOARD_URL=https://dashboard.ceph.unkin.net \ + --from-literal=CEPH_DASHBOARD_USERNAME=k8s-cephrgw-operator \ + --from-literal=CEPH_DASHBOARD_PASSWORD='REPLACE-WITH-A-STRONG-PASSWORD' \ + --from-literal=CEPH_RGW_ENDPOINT=https://s3.ceph.unkin.net +``` + +The deployment carries the `reloader.stakater.com/auto: "true"` annotation, so +rotating this Secret triggers an automatic operator restart — no manual rollout +needed. + +### Sourcing it from Vault (optional) + +If you keep the password in Vault, sync it with a `VaultStaticSecret` (VSO is +already running in `vso-system`) that renders into `cephrgw-credentials` with +the keys above, instead of the plain `kubectl create secret`. The operator does +not care where the Secret comes from, only that those keys exist. + +--- + +## Quick verification + +Once the Secret and dashboard account exist, a smoke test from your workstation: + +```bash +# 1. Log in and capture a token. +TOKEN=$(curl -sk -X POST https://dashboard.ceph.unkin.net/api/auth \ + -H 'Accept: application/vnd.ceph.api.v1.0+json' \ + -H 'Content-Type: application/json' \ + -d '{"username":"k8s-cephrgw-operator","password":"REPLACE-WITH-A-STRONG-PASSWORD"}' \ + | jq -r .token) + +# 2. List RGW users — a 200 with a JSON array means the role + RGW wiring work. +curl -sk https://dashboard.ceph.unkin.net/api/rgw/user \ + -H 'Accept: application/vnd.ceph.api.v1.0+json' \ + -H "Authorization: Bearer $TOKEN" +``` + +If step 1 fails the login/role is wrong (step 1–2 above); if step 1 works but +step 2 returns 500/empty, the dashboard→RGW connection is not configured +(step 2). diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..86953b1 --- /dev/null +++ b/go.mod @@ -0,0 +1,68 @@ +module git.unkin.net/unkin/cephrgw-operator + +go 1.25 + +require ( + k8s.io/api v0.34.4 + k8s.io/apimachinery v0.34.4 + k8s.io/client-go v0.34.4 + sigs.k8s.io/controller-runtime v0.22.4 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2261246 --- /dev/null +++ b/go.sum @@ -0,0 +1,198 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.4 h1:Z5hsoQcZ2yBjelb9j5JKzCVo9qv9XLkVm5llnqS4h+0= +k8s.io/api v0.34.4/go.mod h1:6SaGYuGPkMqqCgg8rPG/OQoCrhgSEV+wWn9v21fDP3o= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.34.4 h1:C5SiSzLEMyWIk53sSbnk0WlOOyqv/MFnWvuc/d6M+xc= +k8s.io/apimachinery v0.34.4/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.4 h1:IXhvzFdm0e897kXtLbeyMpAGzontcShJ/gi/XCCsOLc= +k8s.io/client-go v0.34.4/go.mod h1:tXIVJTQabT5QRGlFdxZQFxrIhcGUPpKL5DAc4gSWTE8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/kind/manifests/namespace.yaml b/hack/kind/manifests/namespace.yaml new file mode 100644 index 0000000..1ad6666 --- /dev/null +++ b/hack/kind/manifests/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cephrgw-system diff --git a/hack/kind/manifests/operator-deployment.yaml b/hack/kind/manifests/operator-deployment.yaml new file mode 100644 index 0000000..95d47ff --- /dev/null +++ b/hack/kind/manifests/operator-deployment.yaml @@ -0,0 +1,74 @@ +--- +# Dashboard credentials for local testing. Replace the values, or create the +# Secret out-of-band, before applying. Keys map 1:1 to the operator env vars. +apiVersion: v1 +kind: Secret +metadata: + name: cephrgw-credentials + namespace: cephrgw-system +type: Opaque +stringData: + CEPH_DASHBOARD_URL: "https://dashboard.ceph.unkin.net" + CEPH_DASHBOARD_USERNAME: "k8s-cephrgw-operator" + CEPH_DASHBOARD_PASSWORD: "change-me" + # Optional: the S3 endpoint written into credential Secrets for consumers. + CEPH_RGW_ENDPOINT: "https://s3.ceph.unkin.net" + # Optional: set to "true" to skip TLS verification (dev only). + # CEPH_DASHBOARD_INSECURE: "true" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cephrgw-operator + namespace: cephrgw-system + labels: + app.kubernetes.io/name: cephrgw-operator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: cephrgw-operator + template: + metadata: + labels: + app.kubernetes.io/name: cephrgw-operator + spec: + serviceAccountName: cephrgw-operator + securityContext: + runAsNonRoot: true + containers: + - name: operator + image: cephrgw-operator:dev + imagePullPolicy: IfNotPresent + args: + - --metrics-bind-address=:8080 + - --health-probe-bind-address=:8081 + - --leader-elect + envFrom: + - secretRef: + name: cephrgw-credentials + ports: + - containerPort: 8080 + name: metrics + - containerPort: 8081 + name: health + readinessProbe: + httpGet: + path: /readyz + port: 8081 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/hack/kind/manifests/rbac.yaml b/hack/kind/manifests/rbac.yaml new file mode 100644 index 0000000..e970390 --- /dev/null +++ b/hack/kind/manifests/rbac.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cephrgw-operator + namespace: cephrgw-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephrgw-operator +rules: + - apiGroups: ["ceph.unkin.net"] + resources: ["*"] + verbs: ["*"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cephrgw-operator +subjects: + - kind: ServiceAccount + name: cephrgw-operator + namespace: cephrgw-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cephrgw-operator diff --git a/internal/ceph/buckets.go b/internal/ceph/buckets.go new file mode 100644 index 0000000..2759d8d --- /dev/null +++ b/internal/ceph/buckets.go @@ -0,0 +1,129 @@ +package ceph + +import ( + "context" + "net/http" + "net/url" + "strconv" +) + +// BucketInfo is the subset of an RGW bucket record the operator consumes. +// Different Ceph releases name the id/name fields slightly differently, so the +// struct captures the known variants and Name/ID normalise them. +type BucketInfo struct { + Bucket string `json:"bucket"` + Bid string `json:"bid"` + ID string `json:"id"` + Owner string `json:"owner"` +} + +// Name returns the bucket name regardless of the field the dashboard used. +func (b *BucketInfo) Name() string { + if b.Bucket != "" { + return b.Bucket + } + return b.Bid +} + +// InstanceID returns the RGW bucket instance id. +func (b *BucketInfo) InstanceID() string { return b.ID } + +// CreateBucketSpec describes a bucket to create. +type CreateBucketSpec struct { + Bucket string + OwnerUID string + Zonegroup string + PlacementTarget string + LockEnabled bool + LockMode string + LockDays *int32 + LockYears *int32 +} + +type createBucketRequest struct { + Bucket string `json:"bucket"` + UID string `json:"uid"` + Zonegroup string `json:"zonegroup,omitempty"` + PlacementTarget string `json:"placement_target,omitempty"` + LockEnabled string `json:"lock_enabled"` + LockMode string `json:"lock_mode,omitempty"` + LockDays string `json:"lock_retention_period_days,omitempty"` + LockYears string `json:"lock_retention_period_years,omitempty"` +} + +// GetBucket fetches a bucket by name, returning an *APIError with status 404 +// (see IsNotFound) when it does not exist. +func (c *Client) GetBucket(ctx context.Context, name string) (*BucketInfo, error) { + var b BucketInfo + if err := c.do(ctx, http.MethodGet, "/api/rgw/bucket/"+url.PathEscape(name), nil, &b, ""); err != nil { + return nil, err + } + return &b, nil +} + +// CreateBucket provisions a bucket owned by spec.OwnerUID. +func (c *Client) CreateBucket(ctx context.Context, spec CreateBucketSpec) (*BucketInfo, error) { + req := createBucketRequest{ + Bucket: spec.Bucket, + UID: spec.OwnerUID, + Zonegroup: spec.Zonegroup, + PlacementTarget: spec.PlacementTarget, + LockEnabled: strconv.FormatBool(spec.LockEnabled), + LockMode: spec.LockMode, + } + if spec.LockDays != nil { + req.LockDays = strconv.Itoa(int(*spec.LockDays)) + } + if spec.LockYears != nil { + req.LockYears = strconv.Itoa(int(*spec.LockYears)) + } + var b BucketInfo + if err := c.do(ctx, http.MethodPost, "/api/rgw/bucket", req, &b, ""); err != nil { + return nil, err + } + return &b, nil +} + +type setBucketRequest struct { + BucketID string `json:"bucket_id"` + UID string `json:"uid"` + VersioningState *string `json:"versioning_state,omitempty"` + BucketPolicy *string `json:"bucket_policy,omitempty"` + Tags *string `json:"tags,omitempty"` +} + +// SetBucketVersioning enables or suspends S3 versioning on a bucket. +func (c *Client) SetBucketVersioning(ctx context.Context, name, bucketID, ownerUID string, enabled bool) error { + state := "Suspended" + if enabled { + state = "Enabled" + } + req := setBucketRequest{BucketID: bucketID, UID: ownerUID, VersioningState: &state} + return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "") +} + +// SetBucketPolicy replaces the S3 bucket policy. An empty policy string asks the +// dashboard to clear it; not every release honours clearing, so callers should +// treat a clear as best-effort. +func (c *Client) SetBucketPolicy(ctx context.Context, name, bucketID, ownerUID, policy string) error { + req := setBucketRequest{BucketID: bucketID, UID: ownerUID, BucketPolicy: &policy} + return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "") +} + +// SetBucketTags replaces the bucket tag set. tagsJSON is the RGW/S3 tag JSON +// (a list of {"Key","Value"} objects). +func (c *Client) SetBucketTags(ctx context.Context, name, bucketID, ownerUID, tagsJSON string) error { + req := setBucketRequest{BucketID: bucketID, UID: ownerUID, Tags: &tagsJSON} + return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "") +} + +// DeleteBucket removes a bucket. When purge is true its objects are deleted too; +// otherwise deletion of a non-empty bucket fails. A 404 is treated as success. +func (c *Client) DeleteBucket(ctx context.Context, name string, purge bool) error { + path := "/api/rgw/bucket/" + url.PathEscape(name) + "?purge_objects=" + strconv.FormatBool(purge) + err := c.do(ctx, http.MethodDelete, path, nil, nil, "") + if IsNotFound(err) { + return nil + } + return err +} diff --git a/internal/ceph/client.go b/internal/ceph/client.go new file mode 100644 index 0000000..8203d06 --- /dev/null +++ b/internal/ceph/client.go @@ -0,0 +1,222 @@ +// Package ceph is a small client for the Ceph manager dashboard REST API, +// scoped to the RGW (S3) user and bucket endpoints the operator needs. +// +// The dashboard authenticates with a username/password to POST /api/auth, which +// returns a bearer (JWT) token. The client caches that token and transparently +// re-authenticates when the server returns 401 (expired/invalid token). +package ceph + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +// defaultAccept is the versioned media type the Ceph dashboard requires on its +// RGW endpoints. The dashboard rejects requests without a matching version. +const defaultAccept = "application/vnd.ceph.api.v1.0+json" + +// Config configures a dashboard Client. +type Config struct { + // BaseURL is the dashboard root, e.g. https://dashboard.ceph.unkin.net. + BaseURL string + // Username / Password authenticate to POST /api/auth. The account needs the + // rgw-manager role (or admin) on the dashboard. + Username string + Password string + // CACert is an optional PEM bundle used to verify the dashboard TLS cert. + CACert []byte + // Insecure disables TLS verification (not recommended). + Insecure bool + // Timeout bounds each HTTP request. Defaults to 30s. + Timeout time.Duration +} + +// Client talks to the Ceph dashboard API. It is safe for concurrent use. +type Client struct { + base string + user string + pass string + http *http.Client + + mu sync.Mutex + token string +} + +// APIError is returned for any non-2xx dashboard response. +type APIError struct { + Status int + Method string + Path string + Body string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("ceph dashboard %s %s: status %d: %s", e.Method, e.Path, e.Status, e.Body) +} + +// IsNotFound reports whether err is a 404 from the dashboard. +func IsNotFound(err error) bool { + var a *APIError + return errors.As(err, &a) && a.Status == http.StatusNotFound +} + +// IsConflict reports whether err is a 409 from the dashboard. +func IsConflict(err error) bool { + var a *APIError + return errors.As(err, &a) && a.Status == http.StatusConflict +} + +// NewClient validates cfg and builds a Client. +func NewClient(cfg Config) (*Client, error) { + base := strings.TrimRight(cfg.BaseURL, "/") + if base == "" { + return nil, fmt.Errorf("ceph: dashboard base URL is required") + } + if cfg.Username == "" || cfg.Password == "" { + return nil, fmt.Errorf("ceph: dashboard username and password are required") + } + + tlsCfg := &tls.Config{InsecureSkipVerify: cfg.Insecure} //nolint:gosec // opt-in via config + if len(cfg.CACert) > 0 { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(cfg.CACert) { + return nil, fmt.Errorf("ceph: failed to parse CA certificate PEM") + } + tlsCfg.RootCAs = pool + } + + timeout := cfg.Timeout + if timeout == 0 { + timeout = 30 * time.Second + } + + return &Client{ + base: base, + user: cfg.Username, + pass: cfg.Password, + http: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + }, + }, nil +} + +// do performs an authenticated request, decoding a 2xx JSON body into out (when +// non-nil). On a 401 it drops the cached token, re-authenticates, and retries +// once. accept overrides the Accept header version when non-empty. +func (c *Client) do(ctx context.Context, method, path string, body, out any, accept string) error { + if accept == "" { + accept = defaultAccept + } + tok, err := c.ensureToken(ctx) + if err != nil { + return err + } + status, err := c.execute(ctx, method, path, body, accept, tok, out) + if status == http.StatusUnauthorized { + c.clearToken() + tok, err = c.ensureToken(ctx) + if err != nil { + return err + } + _, err = c.execute(ctx, method, path, body, accept, tok, out) + } + return err +} + +func (c *Client) ensureToken(ctx context.Context) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token != "" { + return c.token, nil + } + tok, err := c.login(ctx) + if err != nil { + return "", err + } + c.token = tok + return tok, nil +} + +func (c *Client) clearToken() { + c.mu.Lock() + c.token = "" + c.mu.Unlock() +} + +func (c *Client) login(ctx context.Context) (string, error) { + var out struct { + Token string `json:"token"` + } + payload := map[string]string{"username": c.user, "password": c.pass} + if _, err := c.execute(ctx, http.MethodPost, "/api/auth", payload, defaultAccept, "", &out); err != nil { + return "", fmt.Errorf("dashboard login failed: %w", err) + } + if out.Token == "" { + return "", fmt.Errorf("dashboard login returned no token") + } + return out.Token, nil +} + +// execute runs a single request and returns the HTTP status. A non-2xx status +// yields an *APIError. token is sent as a bearer when non-empty. +func (c *Client) execute(ctx context.Context, method, path string, body any, accept, token string, out any) (int, error) { + var reader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return 0, fmt.Errorf("marshal request body: %w", err) + } + reader = bytes.NewReader(b) + } + + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return 0, err + } + req.Header.Set("Accept", accept) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := c.http.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return resp.StatusCode, &APIError{ + Status: resp.StatusCode, + Method: method, + Path: path, + Body: strings.TrimSpace(string(data)), + } + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return resp.StatusCode, fmt.Errorf("decode %s %s response: %w", method, path, err) + } + } + return resp.StatusCode, nil +} + +// Ping verifies connectivity and credentials by authenticating. +func (c *Client) Ping(ctx context.Context) error { + _, err := c.ensureToken(ctx) + return err +} diff --git a/internal/ceph/policy.go b/internal/ceph/policy.go new file mode 100644 index 0000000..7c433f7 --- /dev/null +++ b/internal/ceph/policy.go @@ -0,0 +1,165 @@ +package ceph + +import ( + "encoding/json" + "sort" + "strings" +) + +// Access levels mirrored from the API package to avoid an import cycle; the +// controllers translate their typed level into these strings. +const ( + LevelReadOnly = "read-only" + LevelReadWrite = "read-write" + LevelFull = "full" +) + +// Grant couples an RGW user id with the access level to grant it on a bucket. +type Grant struct { + UID string + Level string +} + +type policyDocument struct { + Version string `json:"Version"` + Statement []policyStatement `json:"Statement"` +} + +type policyStatement struct { + Sid string `json:"Sid"` + Effect string `json:"Effect"` + Principal map[string][]string `json:"Principal"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` +} + +// bucket-level and object-level S3 actions per access level. +var bucketActions = map[string][]string{ + LevelReadOnly: { + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketVersions", + }, + LevelReadWrite: { + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketVersions", + "s3:ListBucketMultipartUploads", + }, +} + +var objectActions = map[string][]string{ + LevelReadOnly: { + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetObjectTagging", + }, + LevelReadWrite: { + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetObjectTagging", + "s3:PutObject", + "s3:PutObjectTagging", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + }, +} + +// BuildBucketPolicy renders a deterministic S3 bucket policy granting each +// principal its requested level. It returns "" when there are no grants so the +// caller can clear the policy. +func BuildBucketPolicy(bucket string, grants []Grant) (string, error) { + if len(grants) == 0 { + return "", nil + } + + sorted := make([]Grant, len(grants)) + copy(sorted, grants) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].UID == sorted[j].UID { + return sorted[i].Level < sorted[j].Level + } + return sorted[i].UID < sorted[j].UID + }) + + bucketARN := "arn:aws:s3:::" + bucket + objectARN := bucketARN + "/*" + + doc := policyDocument{Version: "2012-10-17"} + for _, g := range sorted { + principal := map[string][]string{"AWS": {"arn:aws:iam:::user/" + g.UID}} + switch g.Level { + case LevelFull: + doc.Statement = append(doc.Statement, policyStatement{ + Sid: sid("full", g.UID), + Effect: "Allow", + Principal: principal, + Action: []string{"s3:*"}, + Resource: []string{bucketARN, objectARN}, + }) + default: + doc.Statement = append(doc.Statement, + policyStatement{ + Sid: sid(g.Level+"-bkt", g.UID), + Effect: "Allow", + Principal: principal, + Action: bucketActions[g.Level], + Resource: []string{bucketARN}, + }, + policyStatement{ + Sid: sid(g.Level+"-obj", g.UID), + Effect: "Allow", + Principal: principal, + Action: objectActions[g.Level], + Resource: []string{objectARN}, + }, + ) + } + } + + b, err := json.Marshal(doc) + if err != nil { + return "", err + } + return string(b), nil +} + +// sid builds a policy statement id that only contains characters S3 accepts. +func sid(prefix, uid string) string { + var b strings.Builder + b.WriteString(strings.ReplaceAll(prefix, "-", "")) + for _, r := range uid { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + b.WriteRune(r) + } + } + return b.String() +} + +// BuildTagJSON renders bucket tags in the JSON form the dashboard expects. +func BuildTagJSON(tags map[string]string) (string, error) { + if len(tags) == 0 { + return "", nil + } + keys := make([]string, 0, len(tags)) + for k := range tags { + keys = append(keys, k) + } + sort.Strings(keys) + type kv struct { + Key string `json:"Key"` + Value string `json:"Value"` + } + out := make([]kv, 0, len(keys)) + for _, k := range keys { + out = append(out, kv{Key: k, Value: tags[k]}) + } + b, err := json.Marshal(out) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/internal/ceph/policy_test.go b/internal/ceph/policy_test.go new file mode 100644 index 0000000..0ea18c7 --- /dev/null +++ b/internal/ceph/policy_test.go @@ -0,0 +1,90 @@ +package ceph + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestBuildBucketPolicyEmpty(t *testing.T) { + got, err := BuildBucketPolicy("data", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "" { + t.Fatalf("expected empty policy for no grants, got %q", got) + } +} + +func TestBuildBucketPolicyDeterministic(t *testing.T) { + a, err := BuildBucketPolicy("data", []Grant{ + {UID: "reader", Level: LevelReadOnly}, + {UID: "writer", Level: LevelReadWrite}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + b, err := BuildBucketPolicy("data", []Grant{ + {UID: "writer", Level: LevelReadWrite}, + {UID: "reader", Level: LevelReadOnly}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if a != b { + t.Fatalf("policy is order-dependent:\n a=%s\n b=%s", a, b) + } +} + +func TestBuildBucketPolicyStructure(t *testing.T) { + raw, err := BuildBucketPolicy("data", []Grant{ + {UID: "reader", Level: LevelReadOnly}, + {UID: "admin", Level: LevelFull}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var doc struct { + Version string `json:"Version"` + Statement []struct { + Effect string `json:"Effect"` + Principal map[string][]string `json:"Principal"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` + } `json:"Statement"` + } + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + t.Fatalf("policy is not valid JSON: %v\n%s", err, raw) + } + if doc.Version != "2012-10-17" { + t.Fatalf("unexpected version %q", doc.Version) + } + // read-only -> two statements (bucket + object); full -> one statement. + if len(doc.Statement) != 3 { + t.Fatalf("expected 3 statements, got %d", len(doc.Statement)) + } + + var sawFullWildcard, sawReaderPrincipal bool + for _, s := range doc.Statement { + if s.Effect != "Allow" { + t.Fatalf("expected Allow effect, got %q", s.Effect) + } + for _, a := range s.Action { + if a == "s3:*" { + sawFullWildcard = true + } + } + for _, p := range s.Principal["AWS"] { + if strings.HasSuffix(p, "user/reader") { + sawReaderPrincipal = true + } + } + } + if !sawFullWildcard { + t.Fatal("full grant did not produce an s3:* action") + } + if !sawReaderPrincipal { + t.Fatal("reader principal ARN missing") + } +} diff --git a/internal/ceph/users.go b/internal/ceph/users.go new file mode 100644 index 0000000..d8dc459 --- /dev/null +++ b/internal/ceph/users.go @@ -0,0 +1,152 @@ +package ceph + +import ( + "context" + "net/http" + "net/url" +) + +// UserKey is an S3 access/secret key pair belonging to an RGW user. +type UserKey struct { + User string `json:"user"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` +} + +// User is the subset of an RGW user record the operator consumes. +type User struct { + UID string `json:"user_id"` + DisplayName string `json:"display_name"` + Email string `json:"email"` + MaxBuckets int `json:"max_buckets"` + Suspended int `json:"suspended"` + Keys []UserKey `json:"keys"` +} + +// S3Key returns the first access/secret key pair, if any. +func (u *User) S3Key() (UserKey, bool) { + if len(u.Keys) == 0 { + return UserKey{}, false + } + return u.Keys[0], true +} + +// UserSpec describes the desired state of an RGW user. +type UserSpec struct { + UID string + DisplayName string + Email string + MaxBuckets *int32 + Suspended bool +} + +type createUserRequest struct { + UID string `json:"uid"` + DisplayName string `json:"display_name"` + Email string `json:"email,omitempty"` + MaxBuckets *int32 `json:"max_buckets,omitempty"` + Suspended bool `json:"suspended"` + GenerateKey bool `json:"generate_key"` +} + +type updateUserRequest struct { + DisplayName string `json:"display_name"` + Email string `json:"email,omitempty"` + MaxBuckets *int32 `json:"max_buckets,omitempty"` + Suspended bool `json:"suspended"` +} + +// GetUser fetches an RGW user by uid, returning an *APIError with status 404 +// (see IsNotFound) when it does not exist. +func (c *Client) GetUser(ctx context.Context, uid string) (*User, error) { + var u User + if err := c.do(ctx, http.MethodGet, "/api/rgw/user/"+url.PathEscape(uid), nil, &u, ""); err != nil { + return nil, err + } + return &u, nil +} + +// CreateUser creates an RGW user, asking the dashboard to generate an S3 key +// pair. The returned User carries the generated keys. +func (c *Client) CreateUser(ctx context.Context, spec UserSpec) (*User, error) { + req := createUserRequest{ + UID: spec.UID, + DisplayName: firstNonEmpty(spec.DisplayName, spec.UID), + Email: spec.Email, + MaxBuckets: spec.MaxBuckets, + Suspended: spec.Suspended, + GenerateKey: true, + } + var u User + if err := c.do(ctx, http.MethodPost, "/api/rgw/user", req, &u, ""); err != nil { + return nil, err + } + return &u, nil +} + +// UpdateUser reconciles the mutable attributes of an existing RGW user. +func (c *Client) UpdateUser(ctx context.Context, spec UserSpec) (*User, error) { + req := updateUserRequest{ + DisplayName: firstNonEmpty(spec.DisplayName, spec.UID), + Email: spec.Email, + MaxBuckets: spec.MaxBuckets, + Suspended: spec.Suspended, + } + var u User + if err := c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(spec.UID), req, &u, ""); err != nil { + return nil, err + } + return &u, nil +} + +// DeleteUser removes an RGW user. A 404 is treated as success. +func (c *Client) DeleteUser(ctx context.Context, uid string) error { + err := c.do(ctx, http.MethodDelete, "/api/rgw/user/"+url.PathEscape(uid), nil, nil, "") + if IsNotFound(err) { + return nil + } + return err +} + +type quotaRequest struct { + QuotaType string `json:"quota_type"` + Enabled bool `json:"enabled"` + MaxSizeKb int64 `json:"max_size_kb"` + MaxObjects int64 `json:"max_objects"` +} + +// SetUserQuota applies a quota to a user. quotaType is "user" or "bucket" (the +// latter sets the per-bucket default for buckets the user owns). A nil or +// negative limit means unlimited for that dimension. +func (c *Client) SetUserQuota(ctx context.Context, uid, quotaType string, enabled bool, maxSizeBytes, maxObjects *int64) error { + req := quotaRequest{ + QuotaType: quotaType, + Enabled: enabled, + MaxSizeKb: bytesToKb(maxSizeBytes), + MaxObjects: valueOr(maxObjects, -1), + } + return c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(uid)+"/quota", req, nil, "") +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +func bytesToKb(b *int64) int64 { + if b == nil || *b < 0 { + return -1 + } + return *b / 1024 +} + +func valueOr(v *int64, fallback int64) int64 { + if v == nil || *v < 0 { + return fallback + } + return *v +} diff --git a/internal/controller/bucket_controller.go b/internal/controller/bucket_controller.go new file mode 100644 index 0000000..f7d1d43 --- /dev/null +++ b/internal/controller/bucket_controller.go @@ -0,0 +1,246 @@ +package controller + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1" + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" +) + +// BucketReconciler provisions RGW buckets and owns the bucket's S3 policy. It +// aggregates every BucketAccess that targets the bucket into a single policy +// document, so the policy stays convergent no matter the order of events. +type BucketReconciler struct { + client.Client + Scheme *runtime.Scheme + Ceph *ceph.Client + Endpoint string +} + +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/finalizers,verbs=update + +func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var b v1alpha1.Bucket + if err := r.Get(ctx, req.NamespacedName, &b); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + bucketName := orDefault(b.Spec.BucketName, b.Name) + + if !b.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(&b, finalizer) { + if !b.Spec.RetainOnDelete { + if err := r.Ceph.DeleteBucket(ctx, bucketName, b.Spec.PurgeOnDelete); err != nil { + return r.fail(ctx, &b, "DeleteFailed", err) + } + } + controllerutil.RemoveFinalizer(&b, finalizer) + if err := r.Update(ctx, &b); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + + if controllerutil.AddFinalizer(&b, finalizer) { + if err := r.Update(ctx, &b); err != nil { + return ctrl.Result{}, err + } + } + + // Resolve the owning user. + var owner v1alpha1.ObjectStoreUser + if err := r.Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Spec.OwnerRef}, &owner); err != nil { + if apierrors.IsNotFound(err) { + return r.pending(ctx, &b, "OwnerMissing", fmt.Sprintf("waiting for ObjectStoreUser %q", b.Spec.OwnerRef)) + } + return r.fail(ctx, &b, "OwnerLookupFailed", err) + } + if owner.Status.UID == "" || owner.Status.Phase != "Ready" { + return r.pending(ctx, &b, "OwnerNotReady", fmt.Sprintf("ObjectStoreUser %q not ready", b.Spec.OwnerRef)) + } + ownerUID := owner.Status.UID + + // Ensure the bucket exists. + info, err := r.Ceph.GetBucket(ctx, bucketName) + if ceph.IsNotFound(err) { + createSpec := ceph.CreateBucketSpec{ + Bucket: bucketName, + OwnerUID: ownerUID, + Zonegroup: b.Spec.Zonegroup, + PlacementTarget: b.Spec.PlacementTarget, + } + if ol := b.Spec.ObjectLock; ol != nil && ol.Enabled { + createSpec.LockEnabled = true + createSpec.LockMode = string(ol.Mode) + createSpec.LockDays = ol.Days + createSpec.LockYears = ol.Years + } + info, err = r.Ceph.CreateBucket(ctx, createSpec) + if err != nil { + return r.fail(ctx, &b, "CreateFailed", err) + } + logger.Info("created bucket", "bucket", bucketName, "owner", ownerUID) + } else if err != nil { + return r.fail(ctx, &b, "LookupFailed", err) + } + bucketID := info.InstanceID() + + // Versioning (forced on when object lock is enabled). + if b.Spec.Versioning || (b.Spec.ObjectLock != nil && b.Spec.ObjectLock.Enabled) { + if err := r.Ceph.SetBucketVersioning(ctx, bucketName, bucketID, ownerUID, true); err != nil { + return r.fail(ctx, &b, "VersioningFailed", err) + } + } + + // Tags. + if len(b.Spec.Tags) > 0 { + tj, err := ceph.BuildTagJSON(b.Spec.Tags) + if err != nil { + return r.fail(ctx, &b, "TagsFailed", err) + } + if tj != "" { + if err := r.Ceph.SetBucketTags(ctx, bucketName, bucketID, ownerUID, tj); err != nil { + return r.fail(ctx, &b, "TagsFailed", err) + } + } + } + + // Bucket default quota (applied to the owner). + if q := b.Spec.Quota; q != nil { + if err := r.Ceph.SetUserQuota(ctx, ownerUID, "bucket", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil { + return r.fail(ctx, &b, "QuotaFailed", err) + } + } + + // Render and apply the aggregate S3 policy from all BucketAccess grants. + grants, principals, err := r.collectGrants(ctx, b.Namespace, b.Name) + if err != nil { + return r.fail(ctx, &b, "GrantsFailed", err) + } + policy, err := ceph.BuildBucketPolicy(bucketName, grants) + if err != nil { + return r.fail(ctx, &b, "PolicyBuildFailed", err) + } + if err := r.Ceph.SetBucketPolicy(ctx, bucketName, bucketID, ownerUID, policy); err != nil { + return r.fail(ctx, &b, "PolicyFailed", err) + } + + b.Status.Phase = "Ready" + b.Status.BucketName = bucketName + b.Status.BucketID = bucketID + b.Status.Owner = ownerUID + b.Status.PolicyPrincipals = int32(principals) + b.Status.ObservedGeneration = b.Generation + setReady(&b.Status.Conditions, b.Generation, true, "Provisioned", "bucket provisioned") + if err := r.Status().Update(ctx, &b); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: requeueSteady}, nil +} + +// collectGrants returns the deduplicated set of grants for a bucket, drawn from +// every ready, non-deleting BucketAccess that references it, plus the count of +// distinct principals. +func (r *BucketReconciler) collectGrants(ctx context.Context, namespace, bucketRefName string) ([]ceph.Grant, int, error) { + var list v1alpha1.BucketAccessList + if err := r.List(ctx, &list, client.InNamespace(namespace)); err != nil { + return nil, 0, err + } + seen := map[string]struct{}{} + principals := map[string]struct{}{} + var grants []ceph.Grant + for i := range list.Items { + ba := &list.Items[i] + if ba.Spec.BucketRef != bucketRefName { + continue + } + if !ba.DeletionTimestamp.IsZero() { + continue + } + if ba.Status.UID == "" { + continue + } + key := ba.Status.UID + "|" + string(ba.Spec.Level) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + principals[ba.Status.UID] = struct{}{} + grants = append(grants, ceph.Grant{UID: ba.Status.UID, Level: string(ba.Spec.Level)}) + } + return grants, len(principals), nil +} + +func (r *BucketReconciler) pending(ctx context.Context, b *v1alpha1.Bucket, reason, msg string) (ctrl.Result, error) { + b.Status.Phase = "Pending" + b.Status.ObservedGeneration = b.Generation + setReady(&b.Status.Conditions, b.Generation, false, reason, msg) + if err := r.Status().Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: requeueShort}, nil +} + +func (r *BucketReconciler) fail(ctx context.Context, b *v1alpha1.Bucket, reason string, cause error) (ctrl.Result, error) { + b.Status.Phase = "Error" + b.Status.ObservedGeneration = b.Generation + setReady(&b.Status.Conditions, b.Generation, false, reason, cause.Error()) + if err := r.Status().Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, cause +} + +func (r *BucketReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.Bucket{}). + Watches(&v1alpha1.BucketAccess{}, handler.EnqueueRequestsFromMapFunc(r.bucketForAccess)). + Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.bucketsForOwner)). + Complete(r) +} + +// bucketForAccess maps a BucketAccess change to its referenced Bucket. +func (r *BucketReconciler) bucketForAccess(_ context.Context, obj client.Object) []reconcile.Request { + ba, ok := obj.(*v1alpha1.BucketAccess) + if !ok || ba.Spec.BucketRef == "" { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}}} +} + +// bucketsForOwner maps an ObjectStoreUser change to every Bucket it owns. +func (r *BucketReconciler) bucketsForOwner(ctx context.Context, obj client.Object) []reconcile.Request { + osu, ok := obj.(*v1alpha1.ObjectStoreUser) + if !ok { + return nil + } + var list v1alpha1.BucketList + if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil { + return nil + } + var reqs []reconcile.Request + for i := range list.Items { + if list.Items[i].Spec.OwnerRef == osu.Name { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: list.Items[i].Namespace, Name: list.Items[i].Name, + }}) + } + } + return reqs +} diff --git a/internal/controller/bucketaccess_controller.go b/internal/controller/bucketaccess_controller.go new file mode 100644 index 0000000..56d810b --- /dev/null +++ b/internal/controller/bucketaccess_controller.go @@ -0,0 +1,198 @@ +package controller + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1" + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" +) + +// BucketAccessReconciler ensures the principal for a grant exists (creating a +// dedicated RGW user when none is referenced) and delivers its keys. The bucket +// policy itself is owned and rendered by the Bucket controller, which watches +// BucketAccess objects. +type BucketAccessReconciler struct { + client.Client + Scheme *runtime.Scheme + Ceph *ceph.Client + Endpoint string +} + +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/finalizers,verbs=update + +func (r *BucketAccessReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var ba v1alpha1.BucketAccess + if err := r.Get(ctx, req.NamespacedName, &ba); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + managed := ba.Spec.UserRef == "" + uid, err := r.resolveUID(ctx, &ba) + if err != nil { + return r.fail(ctx, &ba, "ResolveFailed", err) + } + + if !ba.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(&ba, finalizer) { + // Only delete a user the operator created for this grant. + if managed && uid != "" { + if err := r.Ceph.DeleteUser(ctx, uid); err != nil { + return r.fail(ctx, &ba, "DeleteFailed", err) + } + } + controllerutil.RemoveFinalizer(&ba, finalizer) + if err := r.Update(ctx, &ba); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + + if controllerutil.AddFinalizer(&ba, finalizer) { + if err := r.Update(ctx, &ba); err != nil { + return ctrl.Result{}, err + } + } + + // Resolve the referenced Bucket so we can label the credential Secret and + // gate the grant on the bucket existing. + var bucket v1alpha1.Bucket + if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}, &bucket); err != nil { + if apierrors.IsNotFound(err) { + return r.pending(ctx, &ba, "BucketMissing", fmt.Sprintf("waiting for Bucket %q", ba.Spec.BucketRef)) + } + return r.fail(ctx, &ba, "BucketLookupFailed", err) + } + bucketName := orDefault(bucket.Status.BucketName, orDefault(bucket.Spec.BucketName, bucket.Name)) + + secretName := ba.Status.SecretName + if managed { + if uid == "" { + uid = fmt.Sprintf("%s-%s", ba.Spec.BucketRef, ba.Name) + } + secretName = orDefault(ba.Spec.SecretName, ba.Name+"-rgw") + + user, err := r.ensureUser(ctx, uid) + if err != nil { + return r.fail(ctx, &ba, "UserFailed", err) + } + key, ok := user.S3Key() + if !ok { + return r.fail(ctx, &ba, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid)) + } + if err := upsertSecret(ctx, r.Client, r.Scheme, &ba, secretName, ba.Namespace, + credentialSecretData(key, uid, r.Endpoint, bucketName)); err != nil { + return r.fail(ctx, &ba, "SecretFailed", err) + } + } + + ba.Status.Phase = "Ready" + ba.Status.UID = uid + ba.Status.SecretName = secretName + ba.Status.Bound = true + ba.Status.ObservedGeneration = ba.Generation + setReady(&ba.Status.Conditions, ba.Generation, true, "Granted", + fmt.Sprintf("%s access for %s applied to bucket %s", ba.Spec.Level, uid, bucketName)) + if err := r.Status().Update(ctx, &ba); err != nil { + return ctrl.Result{}, err + } + logger.Info("bucket access reconciled", "bucket", bucketName, "uid", uid, "level", ba.Spec.Level) + return ctrl.Result{RequeueAfter: requeueSteady}, nil +} + +// resolveUID returns the RGW uid this grant targets: the referenced +// ObjectStoreUser's provisioned uid, or the managed uid derived from the spec. +func (r *BucketAccessReconciler) resolveUID(ctx context.Context, ba *v1alpha1.BucketAccess) (string, error) { + if ba.Spec.UserRef == "" { + if ba.Spec.UID != "" { + return ba.Spec.UID, nil + } + // Derived lazily in Reconcile once we know it is not a deletion no-op. + return "", nil + } + var osu v1alpha1.ObjectStoreUser + if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.UserRef}, &osu); err != nil { + return "", err + } + if osu.Status.UID == "" { + return "", fmt.Errorf("ObjectStoreUser %q not ready", ba.Spec.UserRef) + } + return osu.Status.UID, nil +} + +func (r *BucketAccessReconciler) ensureUser(ctx context.Context, uid string) (*ceph.User, error) { + if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) { + if _, err := r.Ceph.CreateUser(ctx, ceph.UserSpec{UID: uid, DisplayName: uid}); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return r.Ceph.GetUser(ctx, uid) +} + +func (r *BucketAccessReconciler) pending(ctx context.Context, ba *v1alpha1.BucketAccess, reason, msg string) (ctrl.Result, error) { + ba.Status.Phase = "Pending" + ba.Status.Bound = false + ba.Status.ObservedGeneration = ba.Generation + setReady(&ba.Status.Conditions, ba.Generation, false, reason, msg) + if err := r.Status().Update(ctx, ba); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: requeueShort}, nil +} + +func (r *BucketAccessReconciler) fail(ctx context.Context, ba *v1alpha1.BucketAccess, reason string, cause error) (ctrl.Result, error) { + ba.Status.Phase = "Error" + ba.Status.Bound = false + ba.Status.ObservedGeneration = ba.Generation + setReady(&ba.Status.Conditions, ba.Generation, false, reason, cause.Error()) + if err := r.Status().Update(ctx, ba); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, cause +} + +func (r *BucketAccessReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.BucketAccess{}). + Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.accessForUser)). + Complete(r) +} + +// accessForUser maps an ObjectStoreUser change to every BucketAccess that +// references it, so a grant binds as soon as its user becomes ready. +func (r *BucketAccessReconciler) accessForUser(ctx context.Context, obj client.Object) []reconcile.Request { + osu, ok := obj.(*v1alpha1.ObjectStoreUser) + if !ok { + return nil + } + var list v1alpha1.BucketAccessList + if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil { + return nil + } + var reqs []reconcile.Request + for i := range list.Items { + if list.Items[i].Spec.UserRef == osu.Name { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: list.Items[i].Namespace, Name: list.Items[i].Name, + }}) + } + } + return reqs +} diff --git a/internal/controller/helpers.go b/internal/controller/helpers.go new file mode 100644 index 0000000..07fd58d --- /dev/null +++ b/internal/controller/helpers.go @@ -0,0 +1,101 @@ +package controller + +import ( + "context" + "net/url" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" +) + +// finalizer guards external RGW state (users, buckets, policy statements) so it +// is cleaned up before the Kubernetes object disappears. +const finalizer = "ceph.unkin.net/finalizer" + +// requeueSteady is the resync interval for healthy objects; it lets the +// operator heal drift made directly against RGW. +const requeueSteady = 10 * time.Minute + +// requeueShort backs off on transient "waiting for a dependency" states. +const requeueShort = 30 * time.Second + +// setReady sets the standard Ready condition on a status conditions slice. +func setReady(conds *[]metav1.Condition, gen int64, ok bool, reason, msg string) { + status := metav1.ConditionFalse + if ok { + status = metav1.ConditionTrue + } + meta.SetStatusCondition(conds, metav1.Condition{ + Type: "Ready", + Status: status, + ObservedGeneration: gen, + Reason: reason, + Message: truncate(msg, 32000), + }) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +func orDefault(v, def string) string { + if v != "" { + return v + } + return def +} + +// upsertSecret creates or updates an owner-referenced Opaque Secret with data. +func upsertSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, name, namespace string, data map[string][]byte) error { + sec := &corev1.Secret{} + sec.Name = name + sec.Namespace = namespace + _, err := controllerutil.CreateOrUpdate(ctx, c, sec, func() error { + sec.Type = corev1.SecretTypeOpaque + if sec.Data == nil { + sec.Data = map[string][]byte{} + } + for k, v := range data { + sec.Data[k] = v + } + return controllerutil.SetControllerReference(owner, sec, scheme) + }) + return err +} + +// credentialSecretData assembles the conventional S3/AWS credential keys. +func credentialSecretData(key ceph.UserKey, uid, endpoint, bucket string) map[string][]byte { + data := map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte(key.AccessKey), + "AWS_SECRET_ACCESS_KEY": []byte(key.SecretKey), + "RGW_UID": []byte(uid), + } + if endpoint != "" { + data["S3_ENDPOINT"] = []byte(endpoint) + if host := hostOf(endpoint); host != "" { + data["BUCKET_HOST"] = []byte(host) + } + } + if bucket != "" { + data["BUCKET_NAME"] = []byte(bucket) + } + return data +} + +func hostOf(endpoint string) string { + u, err := url.Parse(endpoint) + if err != nil || u.Host == "" { + return endpoint + } + return u.Host +} diff --git a/internal/controller/objectstoreuser_controller.go b/internal/controller/objectstoreuser_controller.go new file mode 100644 index 0000000..879ecc5 --- /dev/null +++ b/internal/controller/objectstoreuser_controller.go @@ -0,0 +1,126 @@ +package controller + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1" + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" +) + +// ObjectStoreUserReconciler provisions RGW users and delivers their keys. +type ObjectStoreUserReconciler struct { + client.Client + Scheme *runtime.Scheme + Ceph *ceph.Client + Endpoint string +} + +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete + +func (r *ObjectStoreUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var osu v1alpha1.ObjectStoreUser + if err := r.Get(ctx, req.NamespacedName, &osu); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + uid := orDefault(osu.Spec.UID, osu.Name) + secretName := orDefault(osu.Spec.SecretName, osu.Name+"-rgw") + + if !osu.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(&osu, finalizer) { + if err := r.Ceph.DeleteUser(ctx, uid); err != nil { + return r.fail(ctx, &osu, "DeleteFailed", err) + } + controllerutil.RemoveFinalizer(&osu, finalizer) + if err := r.Update(ctx, &osu); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + + if controllerutil.AddFinalizer(&osu, finalizer) { + if err := r.Update(ctx, &osu); err != nil { + return ctrl.Result{}, err + } + } + + spec := ceph.UserSpec{ + UID: uid, + DisplayName: osu.Spec.DisplayName, + Email: osu.Spec.Email, + MaxBuckets: osu.Spec.MaxBuckets, + Suspended: osu.Spec.Suspended, + } + + if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) { + if _, err := r.Ceph.CreateUser(ctx, spec); err != nil { + return r.fail(ctx, &osu, "CreateFailed", err) + } + logger.Info("created RGW user", "uid", uid) + } else if err != nil { + return r.fail(ctx, &osu, "LookupFailed", err) + } else { + if _, err := r.Ceph.UpdateUser(ctx, spec); err != nil { + return r.fail(ctx, &osu, "UpdateFailed", err) + } + } + + if q := osu.Spec.Quota; q != nil { + if err := r.Ceph.SetUserQuota(ctx, uid, "user", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil { + return r.fail(ctx, &osu, "QuotaFailed", err) + } + } + + user, err := r.Ceph.GetUser(ctx, uid) + if err != nil { + return r.fail(ctx, &osu, "LookupFailed", err) + } + key, ok := user.S3Key() + if !ok { + return r.fail(ctx, &osu, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid)) + } + + if err := upsertSecret(ctx, r.Client, r.Scheme, &osu, secretName, osu.Namespace, + credentialSecretData(key, uid, r.Endpoint, "")); err != nil { + return r.fail(ctx, &osu, "SecretFailed", err) + } + + osu.Status.Phase = "Ready" + osu.Status.UID = uid + osu.Status.SecretName = secretName + osu.Status.ObservedGeneration = osu.Generation + setReady(&osu.Status.Conditions, osu.Generation, true, "Provisioned", "RGW user provisioned") + if err := r.Status().Update(ctx, &osu); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: requeueSteady}, nil +} + +func (r *ObjectStoreUserReconciler) fail(ctx context.Context, osu *v1alpha1.ObjectStoreUser, reason string, cause error) (ctrl.Result, error) { + osu.Status.Phase = "Error" + osu.Status.ObservedGeneration = osu.Generation + setReady(&osu.Status.Conditions, osu.Generation, false, reason, cause.Error()) + if err := r.Status().Update(ctx, osu); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, cause +} + +func (r *ObjectStoreUserReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.ObjectStoreUser{}). + Complete(r) +} diff --git a/internal/controller/setup.go b/internal/controller/setup.go new file mode 100644 index 0000000..4c87766 --- /dev/null +++ b/internal/controller/setup.go @@ -0,0 +1,36 @@ +package controller + +import ( + ctrl "sigs.k8s.io/controller-runtime" + + "git.unkin.net/unkin/cephrgw-operator/internal/ceph" +) + +// SetupAll registers every controller with the manager. +func SetupAll(mgr ctrl.Manager, cephClient *ceph.Client, endpoint string) error { + if err := (&ObjectStoreUserReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Ceph: cephClient, + Endpoint: endpoint, + }).SetupWithManager(mgr); err != nil { + return err + } + if err := (&BucketReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Ceph: cephClient, + Endpoint: endpoint, + }).SetupWithManager(mgr); err != nil { + return err + } + if err := (&BucketAccessReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Ceph: cephClient, + Endpoint: endpoint, + }).SetupWithManager(mgr); err != nil { + return err + } + return nil +}