Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a89e38be64 | |||
| ede6604ab1 | |||
| f60e69bb79 | |||
| 5f0b9f53ad | |||
| 7f1f57eb8d | |||
| d3b022d646 | |||
| 343d60cfcf | |||
| c1b3ba1c34 | |||
| 52e183e6f3 | |||
| e7760b79f4 |
@@ -3,8 +3,8 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: docker-build-operator
|
- name: docker-build-operator
|
||||||
image: woodpeckerci/plugin-docker-buildx
|
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||||
settings:
|
settings:
|
||||||
repo: git.unkin.net/unkin/cephrgw-operator
|
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/cephrgw-operator
|
||||||
dockerfile: Dockerfile.operator
|
dockerfile: Dockerfile.operator
|
||||||
dry_run: true
|
dry_run: true
|
||||||
|
|||||||
@@ -4,14 +4,14 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: docker-operator
|
- name: docker-operator
|
||||||
image: woodpeckerci/plugin-docker-buildx
|
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||||
settings:
|
settings:
|
||||||
registry: git.unkin.net
|
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||||
repo: git.unkin.net/unkin/cephrgw-operator
|
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/cephrgw-operator
|
||||||
dockerfile: Dockerfile.operator
|
dockerfile: Dockerfile.operator
|
||||||
username: droneci
|
buildkit_config: |
|
||||||
password:
|
[registry."artifactapi.k8s.syd1.au.unkin.net"]
|
||||||
from_secret: DRONECI_PASSWORD
|
ca = ["/etc/docker/certs.d/artifactapi.k8s.syd1.au.unkin.net/ca.crt"]
|
||||||
tags:
|
tags:
|
||||||
- ${CI_COMMIT_TAG}
|
- ${CI_COMMIT_TAG}
|
||||||
- latest
|
- latest
|
||||||
|
|||||||
@@ -60,6 +60,43 @@ refine it (see `config/samples/04-access-fine-grained.yaml`):
|
|||||||
RGW honours S3 bucket policy on **Reef 18.2+ / Squid**; condition-key support is
|
RGW honours S3 bucket policy on **Reef 18.2+ / Squid**; condition-key support is
|
||||||
a subset of AWS, so validate exotic conditions against your cluster.
|
a subset of AWS, so validate exotic conditions against your cluster.
|
||||||
|
|
||||||
|
### Placement targets
|
||||||
|
|
||||||
|
`Bucket.spec.placementTarget` selects the RGW **placement target** that backs the
|
||||||
|
bucket — i.e. which pools, and therefore which durability profile, store its
|
||||||
|
data. The valid values are cluster configuration, not a fixed set baked into the
|
||||||
|
operator. On this estate radosgw exposes two:
|
||||||
|
|
||||||
|
- `default-placement` — 3× replicated (the cluster default).
|
||||||
|
- `ec` — 4+1 erasure-coded (cheaper capacity, for bulk/archival data).
|
||||||
|
|
||||||
|
Leaving `placementTarget` empty keeps the current behaviour: the owning user's
|
||||||
|
`default_placement` (falling back to the zonegroup default). When set, the
|
||||||
|
operator threads it into the S3 `CreateBucket` `LocationConstraint` as
|
||||||
|
`<zonegroup>:<placementTarget>`; with `spec.zonegroup` empty (the default) that
|
||||||
|
is `:<placementTarget>`, which selects the local/master zonegroup with the given
|
||||||
|
placement — so you do not need to know the zonegroup's api-name to pick a target.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apiVersion: ceph.unkin.net/v1alpha1
|
||||||
|
kind: Bucket
|
||||||
|
metadata:
|
||||||
|
name: raw-archive
|
||||||
|
spec:
|
||||||
|
ownerRef: logarchiver
|
||||||
|
placementTarget: ec # 4+1 erasure-coded pool
|
||||||
|
```
|
||||||
|
|
||||||
|
Placement is **immutable**: RGW fixes it at bucket creation and cannot move an
|
||||||
|
existing bucket between targets. The CRD rejects changing `placementTarget` (and
|
||||||
|
`zonegroup`) on an existing `Bucket`, and if a bucket already lives on a
|
||||||
|
different target than the spec requests (e.g. an adopted bucket, or a value
|
||||||
|
sneaked in around the CRD guard) the controller sets an `Error` phase with a
|
||||||
|
`PlacementImmutable` reason rather than ever deleting and recreating it. The
|
||||||
|
placement RGW actually stores the bucket on is reported in
|
||||||
|
`status.placementTarget` (and the `Placement` print column), so drift is visible.
|
||||||
|
See `config/samples/06-bucket-ec.yaml`.
|
||||||
|
|
||||||
### Adopting existing buckets and users
|
### Adopting existing buckets and users
|
||||||
|
|
||||||
The operator can take over buckets/users that already exist in radosgw and hand
|
The operator can take over buckets/users that already exist in radosgw and hand
|
||||||
@@ -140,7 +177,8 @@ kubectl apply -f hack/kind/manifests/ # edit the Secret first
|
|||||||
|
|
||||||
Woodpecker runs `pre-commit` (gofmt + vet), `test`, and a dry-run image `build`
|
Woodpecker runs `pre-commit` (gofmt + vet), `test`, and a dry-run image `build`
|
||||||
on pull requests; pushing a `v*` tag builds and pushes
|
on pull requests; pushing a `v*` tag builds and pushes
|
||||||
`git.unkin.net/unkin/cephrgw-operator` to the Gitea registry. Bump a release
|
`artifactapi.k8s.syd1.au.unkin.net/docker-internal/cephrgw-operator` to the
|
||||||
|
artifactapi local docker registry. Bump a release
|
||||||
with `make patch|minor|major`.
|
with `make patch|minor|major`.
|
||||||
|
|
||||||
## Notes & caveats
|
## Notes & caveats
|
||||||
|
|||||||
@@ -40,11 +40,29 @@ type BucketSpec struct {
|
|||||||
// with BucketAccess objects.
|
// with BucketAccess objects.
|
||||||
OwnerRef string `json:"ownerRef"`
|
OwnerRef string `json:"ownerRef"`
|
||||||
|
|
||||||
// Zonegroup optionally pins the bucket to a specific RGW zonegroup.
|
// Zonegroup optionally pins the bucket to a specific RGW zonegroup by its
|
||||||
|
// api-name. Empty (the default) uses the cluster's local/master zonegroup, so
|
||||||
|
// PlacementTarget selection works without naming the zonegroup. Immutable:
|
||||||
|
// RGW resolves the zonegroup at bucket creation and cannot move it afterwards.
|
||||||
|
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="zonegroup is immutable; RGW fixes it at bucket creation"
|
||||||
// +optional
|
// +optional
|
||||||
Zonegroup string `json:"zonegroup,omitempty"`
|
Zonegroup string `json:"zonegroup,omitempty"`
|
||||||
|
|
||||||
// PlacementTarget optionally selects a non-default placement target/pool.
|
// PlacementTarget optionally selects the RGW placement target that backs the
|
||||||
|
// bucket, choosing which pools (and thus replication/erasure profile) store
|
||||||
|
// its data. Empty (the default) uses the owning user's default_placement, or
|
||||||
|
// the zonegroup default. The valid values are cluster configuration, not a
|
||||||
|
// fixed set; on this estate the two configured targets are
|
||||||
|
// "default-placement" (3x replicated) and "ec" (4+1 erasure-coded).
|
||||||
|
//
|
||||||
|
// Immutable: RGW chooses the placement at bucket creation (from the S3
|
||||||
|
// LocationConstraint) and cannot move an existing bucket between placement
|
||||||
|
// targets. Set it on a fresh Bucket; changing it later is rejected, and if a
|
||||||
|
// pre-existing bucket is on a different placement the operator reports an
|
||||||
|
// error instead of recreating it.
|
||||||
|
// +kubebuilder:validation:MaxLength=63
|
||||||
|
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$`
|
||||||
|
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="placementTarget is immutable; RGW cannot move a bucket between placement targets"
|
||||||
// +optional
|
// +optional
|
||||||
PlacementTarget string `json:"placementTarget,omitempty"`
|
PlacementTarget string `json:"placementTarget,omitempty"`
|
||||||
|
|
||||||
@@ -100,6 +118,11 @@ type BucketStatus struct {
|
|||||||
// Owner is the RGW uid that owns the bucket.
|
// Owner is the RGW uid that owns the bucket.
|
||||||
// +optional
|
// +optional
|
||||||
Owner string `json:"owner,omitempty"`
|
Owner string `json:"owner,omitempty"`
|
||||||
|
// PlacementTarget is the placement target RGW actually stores the bucket on,
|
||||||
|
// read back from the live bucket. It makes placement drift (a bucket landing
|
||||||
|
// on a different target than spec requested) visible.
|
||||||
|
// +optional
|
||||||
|
PlacementTarget string `json:"placementTarget,omitempty"`
|
||||||
// PolicyPrincipals is the number of extra principals granted via
|
// PolicyPrincipals is the number of extra principals granted via
|
||||||
// BucketAccess and reflected in the bucket policy.
|
// BucketAccess and reflected in the bucket policy.
|
||||||
// +optional
|
// +optional
|
||||||
@@ -121,6 +144,7 @@ type BucketStatus struct {
|
|||||||
// +kubebuilder:resource:shortName=bkt
|
// +kubebuilder:resource:shortName=bkt
|
||||||
// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.status.bucketName`
|
// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.status.bucketName`
|
||||||
// +kubebuilder:printcolumn:name="Owner",type=string,JSONPath=`.status.owner`
|
// +kubebuilder:printcolumn:name="Owner",type=string,JSONPath=`.status.owner`
|
||||||
|
// +kubebuilder:printcolumn:name="Placement",type=string,JSONPath=`.status.placementTarget`
|
||||||
// +kubebuilder:printcolumn:name="Grants",type=integer,JSONPath=`.status.policyPrincipals`
|
// +kubebuilder:printcolumn:name="Grants",type=integer,JSONPath=`.status.policyPrincipals`
|
||||||
// +kubebuilder:printcolumn:name="Adopted",type=boolean,JSONPath=`.status.adopted`
|
// +kubebuilder:printcolumn:name="Adopted",type=boolean,JSONPath=`.status.adopted`
|
||||||
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
|
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
|
||||||
|
|||||||
@@ -73,6 +73,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Advisory only: warn (never exit) if the installed CRDs are missing or
|
||||||
|
// predate this operator's schema, which otherwise surfaces only as opaque
|
||||||
|
// strict-decode failures during reconcile.
|
||||||
|
crdCtx, crdCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
controller.CheckCRDVersions(crdCtx, mgr.GetConfig())
|
||||||
|
crdCancel()
|
||||||
|
|
||||||
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
|
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
|
||||||
logger.Error(err, "unable to set up health check")
|
logger.Error(err, "unable to set up health check")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
@@ -23,6 +23,9 @@ spec:
|
|||||||
- jsonPath: .status.owner
|
- jsonPath: .status.owner
|
||||||
name: Owner
|
name: Owner
|
||||||
type: string
|
type: string
|
||||||
|
- jsonPath: .status.placementTarget
|
||||||
|
name: Placement
|
||||||
|
type: string
|
||||||
- jsonPath: .status.policyPrincipals
|
- jsonPath: .status.policyPrincipals
|
||||||
name: Grants
|
name: Grants
|
||||||
type: integer
|
type: integer
|
||||||
@@ -105,9 +108,26 @@ spec:
|
|||||||
with BucketAccess objects.
|
with BucketAccess objects.
|
||||||
type: string
|
type: string
|
||||||
placementTarget:
|
placementTarget:
|
||||||
description: PlacementTarget optionally selects a non-default placement
|
description: |-
|
||||||
target/pool.
|
PlacementTarget optionally selects the RGW placement target that backs the
|
||||||
|
bucket, choosing which pools (and thus replication/erasure profile) store
|
||||||
|
its data. Empty (the default) uses the owning user's default_placement, or
|
||||||
|
the zonegroup default. The valid values are cluster configuration, not a
|
||||||
|
fixed set; on this estate the two configured targets are
|
||||||
|
"default-placement" (3x replicated) and "ec" (4+1 erasure-coded).
|
||||||
|
|
||||||
|
Immutable: RGW chooses the placement at bucket creation (from the S3
|
||||||
|
LocationConstraint) and cannot move an existing bucket between placement
|
||||||
|
targets. Set it on a fresh Bucket; changing it later is rejected, and if a
|
||||||
|
pre-existing bucket is on a different placement the operator reports an
|
||||||
|
error instead of recreating it.
|
||||||
|
maxLength: 63
|
||||||
|
pattern: ^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$
|
||||||
type: string
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: placementTarget is immutable; RGW cannot move a bucket
|
||||||
|
between placement targets
|
||||||
|
rule: self == oldSelf
|
||||||
purgeOnDelete:
|
purgeOnDelete:
|
||||||
description: |-
|
description: |-
|
||||||
PurgeOnDelete deletes the bucket together with all objects it contains
|
PurgeOnDelete deletes the bucket together with all objects it contains
|
||||||
@@ -148,9 +168,15 @@ spec:
|
|||||||
description: Versioning enables S3 object versioning on the bucket.
|
description: Versioning enables S3 object versioning on the bucket.
|
||||||
type: boolean
|
type: boolean
|
||||||
zonegroup:
|
zonegroup:
|
||||||
description: Zonegroup optionally pins the bucket to a specific RGW
|
description: |-
|
||||||
zonegroup.
|
Zonegroup optionally pins the bucket to a specific RGW zonegroup by its
|
||||||
|
api-name. Empty (the default) uses the cluster's local/master zonegroup, so
|
||||||
|
PlacementTarget selection works without naming the zonegroup. Immutable:
|
||||||
|
RGW resolves the zonegroup at bucket creation and cannot move it afterwards.
|
||||||
type: string
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: zonegroup is immutable; RGW fixes it at bucket creation
|
||||||
|
rule: self == oldSelf
|
||||||
required:
|
required:
|
||||||
- ownerRef
|
- ownerRef
|
||||||
type: object
|
type: object
|
||||||
@@ -236,6 +262,12 @@ spec:
|
|||||||
phase:
|
phase:
|
||||||
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
|
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
|
||||||
type: string
|
type: string
|
||||||
|
placementTarget:
|
||||||
|
description: |-
|
||||||
|
PlacementTarget is the placement target RGW actually stores the bucket on,
|
||||||
|
read back from the live bucket. It makes placement drift (a bucket landing
|
||||||
|
on a different target than spec requested) visible.
|
||||||
|
type: string
|
||||||
policyPrincipals:
|
policyPrincipals:
|
||||||
description: |-
|
description: |-
|
||||||
PolicyPrincipals is the number of extra principals granted via
|
PolicyPrincipals is the number of extra principals granted via
|
||||||
|
|||||||
+36
-4
@@ -293,6 +293,9 @@ spec:
|
|||||||
- jsonPath: .status.owner
|
- jsonPath: .status.owner
|
||||||
name: Owner
|
name: Owner
|
||||||
type: string
|
type: string
|
||||||
|
- jsonPath: .status.placementTarget
|
||||||
|
name: Placement
|
||||||
|
type: string
|
||||||
- jsonPath: .status.policyPrincipals
|
- jsonPath: .status.policyPrincipals
|
||||||
name: Grants
|
name: Grants
|
||||||
type: integer
|
type: integer
|
||||||
@@ -375,9 +378,26 @@ spec:
|
|||||||
with BucketAccess objects.
|
with BucketAccess objects.
|
||||||
type: string
|
type: string
|
||||||
placementTarget:
|
placementTarget:
|
||||||
description: PlacementTarget optionally selects a non-default placement
|
description: |-
|
||||||
target/pool.
|
PlacementTarget optionally selects the RGW placement target that backs the
|
||||||
|
bucket, choosing which pools (and thus replication/erasure profile) store
|
||||||
|
its data. Empty (the default) uses the owning user's default_placement, or
|
||||||
|
the zonegroup default. The valid values are cluster configuration, not a
|
||||||
|
fixed set; on this estate the two configured targets are
|
||||||
|
"default-placement" (3x replicated) and "ec" (4+1 erasure-coded).
|
||||||
|
|
||||||
|
Immutable: RGW chooses the placement at bucket creation (from the S3
|
||||||
|
LocationConstraint) and cannot move an existing bucket between placement
|
||||||
|
targets. Set it on a fresh Bucket; changing it later is rejected, and if a
|
||||||
|
pre-existing bucket is on a different placement the operator reports an
|
||||||
|
error instead of recreating it.
|
||||||
|
maxLength: 63
|
||||||
|
pattern: ^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$
|
||||||
type: string
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: placementTarget is immutable; RGW cannot move a bucket
|
||||||
|
between placement targets
|
||||||
|
rule: self == oldSelf
|
||||||
purgeOnDelete:
|
purgeOnDelete:
|
||||||
description: |-
|
description: |-
|
||||||
PurgeOnDelete deletes the bucket together with all objects it contains
|
PurgeOnDelete deletes the bucket together with all objects it contains
|
||||||
@@ -418,9 +438,15 @@ spec:
|
|||||||
description: Versioning enables S3 object versioning on the bucket.
|
description: Versioning enables S3 object versioning on the bucket.
|
||||||
type: boolean
|
type: boolean
|
||||||
zonegroup:
|
zonegroup:
|
||||||
description: Zonegroup optionally pins the bucket to a specific RGW
|
description: |-
|
||||||
zonegroup.
|
Zonegroup optionally pins the bucket to a specific RGW zonegroup by its
|
||||||
|
api-name. Empty (the default) uses the cluster's local/master zonegroup, so
|
||||||
|
PlacementTarget selection works without naming the zonegroup. Immutable:
|
||||||
|
RGW resolves the zonegroup at bucket creation and cannot move it afterwards.
|
||||||
type: string
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: zonegroup is immutable; RGW fixes it at bucket creation
|
||||||
|
rule: self == oldSelf
|
||||||
required:
|
required:
|
||||||
- ownerRef
|
- ownerRef
|
||||||
type: object
|
type: object
|
||||||
@@ -506,6 +532,12 @@ spec:
|
|||||||
phase:
|
phase:
|
||||||
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
|
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
|
||||||
type: string
|
type: string
|
||||||
|
placementTarget:
|
||||||
|
description: |-
|
||||||
|
PlacementTarget is the placement target RGW actually stores the bucket on,
|
||||||
|
read back from the live bucket. It makes placement drift (a bucket landing
|
||||||
|
on a different target than spec requested) visible.
|
||||||
|
type: string
|
||||||
policyPrincipals:
|
policyPrincipals:
|
||||||
description: |-
|
description: |-
|
||||||
PolicyPrincipals is the number of extra principals granted via
|
PolicyPrincipals is the number of extra principals granted via
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ rules:
|
|||||||
- patch
|
- patch
|
||||||
- update
|
- update
|
||||||
- watch
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apiextensions.k8s.io
|
||||||
|
resources:
|
||||||
|
- customresourcedefinitions
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- ceph.unkin.net
|
- ceph.unkin.net
|
||||||
resources:
|
resources:
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# A bucket placed on the erasure-coded (4+1) placement target instead of the
|
||||||
|
# default 3x-replicated pool. Good for bulk/archival data where capacity matters
|
||||||
|
# more than the extra replica.
|
||||||
|
#
|
||||||
|
# placementTarget is immutable: RGW chooses the placement at bucket creation and
|
||||||
|
# cannot move an existing bucket between targets, so it can only be set on a
|
||||||
|
# fresh Bucket. The operator reports the live placement in status.placementTarget.
|
||||||
|
apiVersion: ceph.unkin.net/v1alpha1
|
||||||
|
kind: Bucket
|
||||||
|
metadata:
|
||||||
|
name: raw-archive
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
bucketName: raw-archive
|
||||||
|
ownerRef: app-owner
|
||||||
|
# Cluster-configured placement target. On this estate: "default-placement"
|
||||||
|
# (3x replicated) or "ec" (4+1 erasure-coded).
|
||||||
|
placementTarget: ec
|
||||||
@@ -9,6 +9,7 @@ require (
|
|||||||
github.com/aws/smithy-go v1.27.4
|
github.com/aws/smithy-go v1.27.4
|
||||||
github.com/ceph/go-ceph v0.40.0
|
github.com/ceph/go-ceph v0.40.0
|
||||||
k8s.io/api v0.34.4
|
k8s.io/api v0.34.4
|
||||||
|
k8s.io/apiextensions-apiserver v0.34.1
|
||||||
k8s.io/apimachinery v0.34.4
|
k8s.io/apimachinery v0.34.4
|
||||||
k8s.io/client-go v0.34.4
|
k8s.io/client-go v0.34.4
|
||||||
sigs.k8s.io/controller-runtime v0.22.4
|
sigs.k8s.io/controller-runtime v0.22.4
|
||||||
@@ -70,7 +71,6 @@ require (
|
|||||||
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
|
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
|
||||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
k8s.io/apiextensions-apiserver v0.34.1 // indirect
|
|
||||||
k8s.io/klog/v2 v2.130.1 // indirect
|
k8s.io/klog/v2 v2.130.1 // indirect
|
||||||
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
|
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
|
||||||
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
|
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
|
||||||
|
|||||||
@@ -18,6 +18,11 @@ type BucketInfo struct {
|
|||||||
Bid string
|
Bid string
|
||||||
ID string
|
ID string
|
||||||
Owner string
|
Owner string
|
||||||
|
// PlacementRule is the placement target RGW stores the bucket on (e.g.
|
||||||
|
// "default-placement" or "ec"), read from the Admin Ops bucket stats.
|
||||||
|
PlacementRule string
|
||||||
|
// Zonegroup is the RGW zonegroup id the bucket belongs to.
|
||||||
|
Zonegroup string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Name returns the bucket name regardless of the field radosgw used.
|
// Name returns the bucket name regardless of the field radosgw used.
|
||||||
@@ -50,7 +55,13 @@ func (c *Client) GetBucket(ctx context.Context, name string) (*BucketInfo, error
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &BucketInfo{Bucket: b.Bucket, ID: b.ID, Owner: b.Owner}, nil
|
return &BucketInfo{
|
||||||
|
Bucket: b.Bucket,
|
||||||
|
ID: b.ID,
|
||||||
|
Owner: b.Owner,
|
||||||
|
PlacementRule: b.PlacementRule,
|
||||||
|
Zonegroup: b.Zonegroup,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateBucket provisions a bucket owned by spec.OwnerUID. The Admin Ops API
|
// CreateBucket provisions a bucket owned by spec.OwnerUID. The Admin Ops API
|
||||||
@@ -211,8 +222,14 @@ func (c *Client) setObjectLockDefault(ctx context.Context, owner func(*s3.Option
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// locationConstraint renders the RGW LocationConstraint from a zonegroup and
|
// locationConstraint renders the RGW S3 CreateBucket LocationConstraint from a
|
||||||
// placement target ("<zonegroup>:<placement>"), or "" for default placement.
|
// zonegroup api-name and a placement target. RGW's S3 create-bucket handler
|
||||||
|
// splits the value on the first ":" — the part before is the zonegroup api-name,
|
||||||
|
// the part after is the placement target id. An empty zonegroup (the common
|
||||||
|
// case) yields ":<placement>", which selects the local/master zonegroup with the
|
||||||
|
// given placement, so callers need not know the zonegroup's api-name to pick a
|
||||||
|
// placement target. Both empty yields "" (no constraint: user/zonegroup
|
||||||
|
// default). Placement empty with a zonegroup set yields just the zonegroup.
|
||||||
func locationConstraint(zonegroup, placement string) string {
|
func locationConstraint(zonegroup, placement string) string {
|
||||||
loc := zonegroup
|
loc := zonegroup
|
||||||
if placement != "" {
|
if placement != "" {
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package ceph
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLocationConstraint(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
zonegroup string
|
||||||
|
placement string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"both empty -> no constraint", "", "", ""},
|
||||||
|
{"placement only -> local zonegroup", "", "ec", ":ec"},
|
||||||
|
{"placement only default target", "", "default-placement", ":default-placement"},
|
||||||
|
{"zonegroup and placement", "default", "ec", "default:ec"},
|
||||||
|
{"zonegroup only", "default", "", "default"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := locationConstraint(tc.zonegroup, tc.placement); got != tc.want {
|
||||||
|
t.Errorf("locationConstraint(%q,%q)=%q want %q", tc.zonegroup, tc.placement, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetBucketPlacement verifies GetBucket surfaces the placement target and
|
||||||
|
// zonegroup from the Admin Ops bucket-stats response, so the controller can
|
||||||
|
// detect placement drift.
|
||||||
|
func TestGetBucketPlacement(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"bucket": "raw-archive",
|
||||||
|
"id": "eae688bc-ee35-445d-9188-111b73c8b4a0.12345.1",
|
||||||
|
"owner": "logarchiver",
|
||||||
|
"zonegroup": "eae688bc-ee35-445d-9188-111b73c8b4a0",
|
||||||
|
"placement_rule": "ec"
|
||||||
|
}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c, err := NewClient(Config{Endpoint: srv.URL, AccessKey: "a", SecretKey: "s"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewClient: %v", err)
|
||||||
|
}
|
||||||
|
info, err := c.GetBucket(context.Background(), "raw-archive")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetBucket: %v", err)
|
||||||
|
}
|
||||||
|
if info.PlacementRule != "ec" {
|
||||||
|
t.Errorf("PlacementRule=%q want %q", info.PlacementRule, "ec")
|
||||||
|
}
|
||||||
|
if info.Zonegroup != "eae688bc-ee35-445d-9188-111b73c8b4a0" {
|
||||||
|
t.Errorf("Zonegroup=%q unexpected", info.Zonegroup)
|
||||||
|
}
|
||||||
|
if info.Owner != "logarchiver" {
|
||||||
|
t.Errorf("Owner=%q want logarchiver", info.Owner)
|
||||||
|
}
|
||||||
|
if info.Name() != "raw-archive" {
|
||||||
|
t.Errorf("Name()=%q want raw-archive", info.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -112,6 +112,18 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
|
|||||||
}
|
}
|
||||||
bucketID := info.InstanceID()
|
bucketID := info.InstanceID()
|
||||||
|
|
||||||
|
// Placement is fixed at creation: RGW cannot move an existing bucket between
|
||||||
|
// placement targets. If the live bucket sits on a different target than the
|
||||||
|
// spec asks for (a changed spec, or an adopted bucket that predates the
|
||||||
|
// request), surface a clear error instead of ever deleting/recreating it. An
|
||||||
|
// empty PlacementTarget imposes no constraint.
|
||||||
|
if pc := placementConflict(b.Spec.PlacementTarget, info.PlacementRule); pc {
|
||||||
|
b.Status.PlacementTarget = info.PlacementRule
|
||||||
|
return r.fail(ctx, &b, "PlacementImmutable", fmt.Errorf(
|
||||||
|
"bucket %q is on placement target %q but spec requests %q; RGW cannot move a bucket between placement targets",
|
||||||
|
bucketName, info.PlacementRule, b.Spec.PlacementTarget))
|
||||||
|
}
|
||||||
|
|
||||||
// Versioning (forced on when object lock is enabled).
|
// Versioning (forced on when object lock is enabled).
|
||||||
if b.Spec.Versioning || (b.Spec.ObjectLock != nil && b.Spec.ObjectLock.Enabled) {
|
if b.Spec.Versioning || (b.Spec.ObjectLock != nil && b.Spec.ObjectLock.Enabled) {
|
||||||
if err := r.Ceph.SetBucketVersioning(ctx, bucketName, bucketID, ownerUID, true); err != nil {
|
if err := r.Ceph.SetBucketVersioning(ctx, bucketName, bucketID, ownerUID, true); err != nil {
|
||||||
@@ -167,6 +179,7 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
|
|||||||
b.Status.BucketName = bucketName
|
b.Status.BucketName = bucketName
|
||||||
b.Status.BucketID = bucketID
|
b.Status.BucketID = bucketID
|
||||||
b.Status.Owner = ownerUID
|
b.Status.Owner = ownerUID
|
||||||
|
b.Status.PlacementTarget = info.PlacementRule
|
||||||
b.Status.PolicyPrincipals = int32(principals)
|
b.Status.PolicyPrincipals = int32(principals)
|
||||||
b.Status.ObservedGeneration = b.Generation
|
b.Status.ObservedGeneration = b.Generation
|
||||||
setReady(&b.Status.Conditions, b.Generation, true, "Provisioned", "bucket provisioned")
|
setReady(&b.Status.Conditions, b.Generation, true, "Provisioned", "bucket provisioned")
|
||||||
@@ -250,6 +263,17 @@ func managePolicy(b *v1alpha1.Bucket) bool {
|
|||||||
return b.Spec.ManagePolicy == nil || *b.Spec.ManagePolicy
|
return b.Spec.ManagePolicy == nil || *b.Spec.ManagePolicy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// placementConflict reports whether a bucket's live placement target violates
|
||||||
|
// the spec. An empty spec placement imposes no constraint (the bucket may sit on
|
||||||
|
// whatever default it was created with). Otherwise the live placement must match
|
||||||
|
// exactly, since RGW cannot move a bucket between placement targets.
|
||||||
|
func placementConflict(specPlacement, livePlacement string) bool {
|
||||||
|
if specPlacement == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return specPlacement != livePlacement
|
||||||
|
}
|
||||||
|
|
||||||
func (r *BucketReconciler) pending(ctx context.Context, b *v1alpha1.Bucket, reason, msg string) (ctrl.Result, error) {
|
func (r *BucketReconciler) pending(ctx context.Context, b *v1alpha1.Bucket, reason, msg string) (ctrl.Result, error) {
|
||||||
b.Status.Phase = "Pending"
|
b.Status.Phase = "Pending"
|
||||||
b.Status.ObservedGeneration = b.Generation
|
b.Status.ObservedGeneration = b.Generation
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package controller
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestPlacementConflict(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
specPlacement string
|
||||||
|
livePlacement string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"unset spec never conflicts", "", "default-placement", false},
|
||||||
|
{"unset spec unset live", "", "", false},
|
||||||
|
{"matching ec", "ec", "ec", false},
|
||||||
|
{"matching default", "default-placement", "default-placement", false},
|
||||||
|
{"ec requested but default live", "ec", "default-placement", true},
|
||||||
|
{"default requested but ec live", "default-placement", "ec", true},
|
||||||
|
{"spec set live empty", "ec", "", true},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := placementConflict(tc.specPlacement, tc.livePlacement); got != tc.want {
|
||||||
|
t.Errorf("placementConflict(%q,%q)=%v want %v", tc.specPlacement, tc.livePlacement, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package controller
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
|
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/rest"
|
||||||
|
ctrl "sigs.k8s.io/controller-runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The CRD version check reads CustomResourceDefinitions to compare the
|
||||||
|
// installed schema against what this operator expects.
|
||||||
|
//
|
||||||
|
// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list
|
||||||
|
|
||||||
|
// crdSentinel names a CRD and a spec property that only exists in the schema
|
||||||
|
// version shipped alongside this operator build. If the installed CRD lacks the
|
||||||
|
// sentinel, its schema predates this operator and strict decoding of new spec
|
||||||
|
// fields will silently fail. Keep this list in one place so new fields are easy
|
||||||
|
// to register as sentinels.
|
||||||
|
type crdSentinel struct {
|
||||||
|
// crd is the metadata.name of the CustomResourceDefinition.
|
||||||
|
crd string
|
||||||
|
// specProperty is a key expected under
|
||||||
|
// .spec.versions[].schema.openAPIV3Schema.properties.spec.properties.
|
||||||
|
specProperty string
|
||||||
|
}
|
||||||
|
|
||||||
|
// crdSentinels is the authoritative list checked at startup. Extend it whenever
|
||||||
|
// a new spec field is added that older CRDs would reject.
|
||||||
|
var crdSentinels = []crdSentinel{
|
||||||
|
{crd: "buckets.ceph.unkin.net", specProperty: "managePolicy"},
|
||||||
|
{crd: "objectstoreusers.ceph.unkin.net", specProperty: "retainOnDelete"},
|
||||||
|
{crd: "bucketaccesses.ceph.unkin.net", specProperty: "rawStatements"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckCRDVersions verifies that every CRD this operator owns is installed and
|
||||||
|
// carries the schema fields this build expects. It is advisory only: it logs a
|
||||||
|
// distinct WARNING per problem and never returns an error or exits, so a stale
|
||||||
|
// or missing CRD cannot block startup.
|
||||||
|
func CheckCRDVersions(ctx context.Context, cfg *rest.Config) {
|
||||||
|
log := ctrl.Log.WithName("crd-version-check")
|
||||||
|
|
||||||
|
client, err := apiextensionsclient.NewForConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err, "unable to build apiextensions client; skipping CRD version check")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, s := range crdSentinels {
|
||||||
|
crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, s.crd, metav1.GetOptions{})
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
log.Info("WARNING: CRD is not installed — apply the CRDs matching this operator version",
|
||||||
|
"crd", s.crd)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err, "unable to read CRD; cannot verify it matches this operator version",
|
||||||
|
"crd", s.crd)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !crdHasSpecProperty(crd, s.specProperty) {
|
||||||
|
log.Info("WARNING: CRD is out of date — apply the CRDs matching this operator version",
|
||||||
|
"crd", s.crd, "missingField", "spec."+s.specProperty)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// crdHasSpecProperty reports whether any served/stored version of the CRD
|
||||||
|
// declares the given property under spec.
|
||||||
|
func crdHasSpecProperty(crd *apiextensionsv1.CustomResourceDefinition, property string) bool {
|
||||||
|
for _, v := range crd.Spec.Versions {
|
||||||
|
schema := v.Schema
|
||||||
|
if schema == nil || schema.OpenAPIV3Schema == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
specSchema, ok := schema.OpenAPIV3Schema.Properties["spec"]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := specSchema.Properties[property]; ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user