Initial cephrgw-operator: Ceph RGW buckets & keys via dashboard API
Adds a Kubernetes operator that provisions Ceph RGW (S3) buckets and access keys declaratively through the Ceph manager dashboard REST API. Three CRDs in group ceph.unkin.net/v1alpha1: - ObjectStoreUser: creates an RGW user, delivers its key pair to a Secret - Bucket: creates an S3 bucket owned by an ObjectStoreUser; owns the bucket's aggregate S3 policy (union of all BucketAccess grants) - BucketAccess: grants read-only/read-write/full access, provisioning a dedicated user (or reusing a referenced one) and delivering RW/RO keys The internal/ceph client wraps the dashboard /api/auth, /api/rgw/user and /api/rgw/bucket endpoints with lazy token auth and re-auth on 401. Bucket policies are rendered deterministically and applied via the bucket policy API (Reef 18.2+). Credentials come from the cephrgw-credentials Secret via env. Includes generated CRDs/RBAC, samples, kind manifests, Woodpecker CI, and docs/ceph-setup.md covering the required Ceph dashboard account, RGW wiring and permissions.
This commit is contained in:
benvin
@@ -0,0 +1,129 @@
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// BucketInfo is the subset of an RGW bucket record the operator consumes.
|
||||
// Different Ceph releases name the id/name fields slightly differently, so the
|
||||
// struct captures the known variants and Name/ID normalise them.
|
||||
type BucketInfo struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Bid string `json:"bid"`
|
||||
ID string `json:"id"`
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
|
||||
// Name returns the bucket name regardless of the field the dashboard used.
|
||||
func (b *BucketInfo) Name() string {
|
||||
if b.Bucket != "" {
|
||||
return b.Bucket
|
||||
}
|
||||
return b.Bid
|
||||
}
|
||||
|
||||
// InstanceID returns the RGW bucket instance id.
|
||||
func (b *BucketInfo) InstanceID() string { return b.ID }
|
||||
|
||||
// CreateBucketSpec describes a bucket to create.
|
||||
type CreateBucketSpec struct {
|
||||
Bucket string
|
||||
OwnerUID string
|
||||
Zonegroup string
|
||||
PlacementTarget string
|
||||
LockEnabled bool
|
||||
LockMode string
|
||||
LockDays *int32
|
||||
LockYears *int32
|
||||
}
|
||||
|
||||
type createBucketRequest struct {
|
||||
Bucket string `json:"bucket"`
|
||||
UID string `json:"uid"`
|
||||
Zonegroup string `json:"zonegroup,omitempty"`
|
||||
PlacementTarget string `json:"placement_target,omitempty"`
|
||||
LockEnabled string `json:"lock_enabled"`
|
||||
LockMode string `json:"lock_mode,omitempty"`
|
||||
LockDays string `json:"lock_retention_period_days,omitempty"`
|
||||
LockYears string `json:"lock_retention_period_years,omitempty"`
|
||||
}
|
||||
|
||||
// GetBucket fetches a bucket by name, returning an *APIError with status 404
|
||||
// (see IsNotFound) when it does not exist.
|
||||
func (c *Client) GetBucket(ctx context.Context, name string) (*BucketInfo, error) {
|
||||
var b BucketInfo
|
||||
if err := c.do(ctx, http.MethodGet, "/api/rgw/bucket/"+url.PathEscape(name), nil, &b, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// CreateBucket provisions a bucket owned by spec.OwnerUID.
|
||||
func (c *Client) CreateBucket(ctx context.Context, spec CreateBucketSpec) (*BucketInfo, error) {
|
||||
req := createBucketRequest{
|
||||
Bucket: spec.Bucket,
|
||||
UID: spec.OwnerUID,
|
||||
Zonegroup: spec.Zonegroup,
|
||||
PlacementTarget: spec.PlacementTarget,
|
||||
LockEnabled: strconv.FormatBool(spec.LockEnabled),
|
||||
LockMode: spec.LockMode,
|
||||
}
|
||||
if spec.LockDays != nil {
|
||||
req.LockDays = strconv.Itoa(int(*spec.LockDays))
|
||||
}
|
||||
if spec.LockYears != nil {
|
||||
req.LockYears = strconv.Itoa(int(*spec.LockYears))
|
||||
}
|
||||
var b BucketInfo
|
||||
if err := c.do(ctx, http.MethodPost, "/api/rgw/bucket", req, &b, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
type setBucketRequest struct {
|
||||
BucketID string `json:"bucket_id"`
|
||||
UID string `json:"uid"`
|
||||
VersioningState *string `json:"versioning_state,omitempty"`
|
||||
BucketPolicy *string `json:"bucket_policy,omitempty"`
|
||||
Tags *string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// SetBucketVersioning enables or suspends S3 versioning on a bucket.
|
||||
func (c *Client) SetBucketVersioning(ctx context.Context, name, bucketID, ownerUID string, enabled bool) error {
|
||||
state := "Suspended"
|
||||
if enabled {
|
||||
state = "Enabled"
|
||||
}
|
||||
req := setBucketRequest{BucketID: bucketID, UID: ownerUID, VersioningState: &state}
|
||||
return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "")
|
||||
}
|
||||
|
||||
// SetBucketPolicy replaces the S3 bucket policy. An empty policy string asks the
|
||||
// dashboard to clear it; not every release honours clearing, so callers should
|
||||
// treat a clear as best-effort.
|
||||
func (c *Client) SetBucketPolicy(ctx context.Context, name, bucketID, ownerUID, policy string) error {
|
||||
req := setBucketRequest{BucketID: bucketID, UID: ownerUID, BucketPolicy: &policy}
|
||||
return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "")
|
||||
}
|
||||
|
||||
// SetBucketTags replaces the bucket tag set. tagsJSON is the RGW/S3 tag JSON
|
||||
// (a list of {"Key","Value"} objects).
|
||||
func (c *Client) SetBucketTags(ctx context.Context, name, bucketID, ownerUID, tagsJSON string) error {
|
||||
req := setBucketRequest{BucketID: bucketID, UID: ownerUID, Tags: &tagsJSON}
|
||||
return c.do(ctx, http.MethodPut, "/api/rgw/bucket/"+url.PathEscape(name), req, nil, "")
|
||||
}
|
||||
|
||||
// DeleteBucket removes a bucket. When purge is true its objects are deleted too;
|
||||
// otherwise deletion of a non-empty bucket fails. A 404 is treated as success.
|
||||
func (c *Client) DeleteBucket(ctx context.Context, name string, purge bool) error {
|
||||
path := "/api/rgw/bucket/" + url.PathEscape(name) + "?purge_objects=" + strconv.FormatBool(purge)
|
||||
err := c.do(ctx, http.MethodDelete, path, nil, nil, "")
|
||||
if IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
// Package ceph is a small client for the Ceph manager dashboard REST API,
|
||||
// scoped to the RGW (S3) user and bucket endpoints the operator needs.
|
||||
//
|
||||
// The dashboard authenticates with a username/password to POST /api/auth, which
|
||||
// returns a bearer (JWT) token. The client caches that token and transparently
|
||||
// re-authenticates when the server returns 401 (expired/invalid token).
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// defaultAccept is the versioned media type the Ceph dashboard requires on its
|
||||
// RGW endpoints. The dashboard rejects requests without a matching version.
|
||||
const defaultAccept = "application/vnd.ceph.api.v1.0+json"
|
||||
|
||||
// Config configures a dashboard Client.
|
||||
type Config struct {
|
||||
// BaseURL is the dashboard root, e.g. https://dashboard.ceph.unkin.net.
|
||||
BaseURL string
|
||||
// Username / Password authenticate to POST /api/auth. The account needs the
|
||||
// rgw-manager role (or admin) on the dashboard.
|
||||
Username string
|
||||
Password string
|
||||
// CACert is an optional PEM bundle used to verify the dashboard TLS cert.
|
||||
CACert []byte
|
||||
// Insecure disables TLS verification (not recommended).
|
||||
Insecure bool
|
||||
// Timeout bounds each HTTP request. Defaults to 30s.
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Client talks to the Ceph dashboard API. It is safe for concurrent use.
|
||||
type Client struct {
|
||||
base string
|
||||
user string
|
||||
pass string
|
||||
http *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
token string
|
||||
}
|
||||
|
||||
// APIError is returned for any non-2xx dashboard response.
|
||||
type APIError struct {
|
||||
Status int
|
||||
Method string
|
||||
Path string
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string {
|
||||
return fmt.Sprintf("ceph dashboard %s %s: status %d: %s", e.Method, e.Path, e.Status, e.Body)
|
||||
}
|
||||
|
||||
// IsNotFound reports whether err is a 404 from the dashboard.
|
||||
func IsNotFound(err error) bool {
|
||||
var a *APIError
|
||||
return errors.As(err, &a) && a.Status == http.StatusNotFound
|
||||
}
|
||||
|
||||
// IsConflict reports whether err is a 409 from the dashboard.
|
||||
func IsConflict(err error) bool {
|
||||
var a *APIError
|
||||
return errors.As(err, &a) && a.Status == http.StatusConflict
|
||||
}
|
||||
|
||||
// NewClient validates cfg and builds a Client.
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
base := strings.TrimRight(cfg.BaseURL, "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("ceph: dashboard base URL is required")
|
||||
}
|
||||
if cfg.Username == "" || cfg.Password == "" {
|
||||
return nil, fmt.Errorf("ceph: dashboard username and password are required")
|
||||
}
|
||||
|
||||
tlsCfg := &tls.Config{InsecureSkipVerify: cfg.Insecure} //nolint:gosec // opt-in via config
|
||||
if len(cfg.CACert) > 0 {
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(cfg.CACert) {
|
||||
return nil, fmt.Errorf("ceph: failed to parse CA certificate PEM")
|
||||
}
|
||||
tlsCfg.RootCAs = pool
|
||||
}
|
||||
|
||||
timeout := cfg.Timeout
|
||||
if timeout == 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
return &Client{
|
||||
base: base,
|
||||
user: cfg.Username,
|
||||
pass: cfg.Password,
|
||||
http: &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: &http.Transport{TLSClientConfig: tlsCfg},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// do performs an authenticated request, decoding a 2xx JSON body into out (when
|
||||
// non-nil). On a 401 it drops the cached token, re-authenticates, and retries
|
||||
// once. accept overrides the Accept header version when non-empty.
|
||||
func (c *Client) do(ctx context.Context, method, path string, body, out any, accept string) error {
|
||||
if accept == "" {
|
||||
accept = defaultAccept
|
||||
}
|
||||
tok, err := c.ensureToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := c.execute(ctx, method, path, body, accept, tok, out)
|
||||
if status == http.StatusUnauthorized {
|
||||
c.clearToken()
|
||||
tok, err = c.ensureToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = c.execute(ctx, method, path, body, accept, tok, out)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Client) ensureToken(ctx context.Context) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.token != "" {
|
||||
return c.token, nil
|
||||
}
|
||||
tok, err := c.login(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
c.token = tok
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
func (c *Client) clearToken() {
|
||||
c.mu.Lock()
|
||||
c.token = ""
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *Client) login(ctx context.Context) (string, error) {
|
||||
var out struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
payload := map[string]string{"username": c.user, "password": c.pass}
|
||||
if _, err := c.execute(ctx, http.MethodPost, "/api/auth", payload, defaultAccept, "", &out); err != nil {
|
||||
return "", fmt.Errorf("dashboard login failed: %w", err)
|
||||
}
|
||||
if out.Token == "" {
|
||||
return "", fmt.Errorf("dashboard login returned no token")
|
||||
}
|
||||
return out.Token, nil
|
||||
}
|
||||
|
||||
// execute runs a single request and returns the HTTP status. A non-2xx status
|
||||
// yields an *APIError. token is sent as a bearer when non-empty.
|
||||
func (c *Client) execute(ctx context.Context, method, path string, body any, accept, token string, out any) (int, error) {
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
reader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return resp.StatusCode, &APIError{
|
||||
Status: resp.StatusCode,
|
||||
Method: method,
|
||||
Path: path,
|
||||
Body: strings.TrimSpace(string(data)),
|
||||
}
|
||||
}
|
||||
if out != nil && len(data) > 0 {
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return resp.StatusCode, fmt.Errorf("decode %s %s response: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
// Ping verifies connectivity and credentials by authenticating.
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
_, err := c.ensureToken(ctx)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Access levels mirrored from the API package to avoid an import cycle; the
|
||||
// controllers translate their typed level into these strings.
|
||||
const (
|
||||
LevelReadOnly = "read-only"
|
||||
LevelReadWrite = "read-write"
|
||||
LevelFull = "full"
|
||||
)
|
||||
|
||||
// Grant couples an RGW user id with the access level to grant it on a bucket.
|
||||
type Grant struct {
|
||||
UID string
|
||||
Level string
|
||||
}
|
||||
|
||||
type policyDocument struct {
|
||||
Version string `json:"Version"`
|
||||
Statement []policyStatement `json:"Statement"`
|
||||
}
|
||||
|
||||
type policyStatement struct {
|
||||
Sid string `json:"Sid"`
|
||||
Effect string `json:"Effect"`
|
||||
Principal map[string][]string `json:"Principal"`
|
||||
Action []string `json:"Action"`
|
||||
Resource []string `json:"Resource"`
|
||||
}
|
||||
|
||||
// bucket-level and object-level S3 actions per access level.
|
||||
var bucketActions = map[string][]string{
|
||||
LevelReadOnly: {
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucketVersions",
|
||||
},
|
||||
LevelReadWrite: {
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucketVersions",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
},
|
||||
}
|
||||
|
||||
var objectActions = map[string][]string{
|
||||
LevelReadOnly: {
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion",
|
||||
"s3:GetObjectTagging",
|
||||
},
|
||||
LevelReadWrite: {
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion",
|
||||
"s3:GetObjectTagging",
|
||||
"s3:PutObject",
|
||||
"s3:PutObjectTagging",
|
||||
"s3:DeleteObject",
|
||||
"s3:DeleteObjectVersion",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
},
|
||||
}
|
||||
|
||||
// BuildBucketPolicy renders a deterministic S3 bucket policy granting each
|
||||
// principal its requested level. It returns "" when there are no grants so the
|
||||
// caller can clear the policy.
|
||||
func BuildBucketPolicy(bucket string, grants []Grant) (string, error) {
|
||||
if len(grants) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
sorted := make([]Grant, len(grants))
|
||||
copy(sorted, grants)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].UID == sorted[j].UID {
|
||||
return sorted[i].Level < sorted[j].Level
|
||||
}
|
||||
return sorted[i].UID < sorted[j].UID
|
||||
})
|
||||
|
||||
bucketARN := "arn:aws:s3:::" + bucket
|
||||
objectARN := bucketARN + "/*"
|
||||
|
||||
doc := policyDocument{Version: "2012-10-17"}
|
||||
for _, g := range sorted {
|
||||
principal := map[string][]string{"AWS": {"arn:aws:iam:::user/" + g.UID}}
|
||||
switch g.Level {
|
||||
case LevelFull:
|
||||
doc.Statement = append(doc.Statement, policyStatement{
|
||||
Sid: sid("full", g.UID),
|
||||
Effect: "Allow",
|
||||
Principal: principal,
|
||||
Action: []string{"s3:*"},
|
||||
Resource: []string{bucketARN, objectARN},
|
||||
})
|
||||
default:
|
||||
doc.Statement = append(doc.Statement,
|
||||
policyStatement{
|
||||
Sid: sid(g.Level+"-bkt", g.UID),
|
||||
Effect: "Allow",
|
||||
Principal: principal,
|
||||
Action: bucketActions[g.Level],
|
||||
Resource: []string{bucketARN},
|
||||
},
|
||||
policyStatement{
|
||||
Sid: sid(g.Level+"-obj", g.UID),
|
||||
Effect: "Allow",
|
||||
Principal: principal,
|
||||
Action: objectActions[g.Level],
|
||||
Resource: []string{objectARN},
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
// sid builds a policy statement id that only contains characters S3 accepts.
|
||||
func sid(prefix, uid string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(strings.ReplaceAll(prefix, "-", ""))
|
||||
for _, r := range uid {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// BuildTagJSON renders bucket tags in the JSON form the dashboard expects.
|
||||
func BuildTagJSON(tags map[string]string) (string, error) {
|
||||
if len(tags) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
keys := make([]string, 0, len(tags))
|
||||
for k := range tags {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
type kv struct {
|
||||
Key string `json:"Key"`
|
||||
Value string `json:"Value"`
|
||||
}
|
||||
out := make([]kv, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, kv{Key: k, Value: tags[k]})
|
||||
}
|
||||
b, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildBucketPolicyEmpty(t *testing.T) {
|
||||
got, err := BuildBucketPolicy("data", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("expected empty policy for no grants, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBucketPolicyDeterministic(t *testing.T) {
|
||||
a, err := BuildBucketPolicy("data", []Grant{
|
||||
{UID: "reader", Level: LevelReadOnly},
|
||||
{UID: "writer", Level: LevelReadWrite},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
b, err := BuildBucketPolicy("data", []Grant{
|
||||
{UID: "writer", Level: LevelReadWrite},
|
||||
{UID: "reader", Level: LevelReadOnly},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if a != b {
|
||||
t.Fatalf("policy is order-dependent:\n a=%s\n b=%s", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBucketPolicyStructure(t *testing.T) {
|
||||
raw, err := BuildBucketPolicy("data", []Grant{
|
||||
{UID: "reader", Level: LevelReadOnly},
|
||||
{UID: "admin", Level: LevelFull},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
var doc struct {
|
||||
Version string `json:"Version"`
|
||||
Statement []struct {
|
||||
Effect string `json:"Effect"`
|
||||
Principal map[string][]string `json:"Principal"`
|
||||
Action []string `json:"Action"`
|
||||
Resource []string `json:"Resource"`
|
||||
} `json:"Statement"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &doc); err != nil {
|
||||
t.Fatalf("policy is not valid JSON: %v\n%s", err, raw)
|
||||
}
|
||||
if doc.Version != "2012-10-17" {
|
||||
t.Fatalf("unexpected version %q", doc.Version)
|
||||
}
|
||||
// read-only -> two statements (bucket + object); full -> one statement.
|
||||
if len(doc.Statement) != 3 {
|
||||
t.Fatalf("expected 3 statements, got %d", len(doc.Statement))
|
||||
}
|
||||
|
||||
var sawFullWildcard, sawReaderPrincipal bool
|
||||
for _, s := range doc.Statement {
|
||||
if s.Effect != "Allow" {
|
||||
t.Fatalf("expected Allow effect, got %q", s.Effect)
|
||||
}
|
||||
for _, a := range s.Action {
|
||||
if a == "s3:*" {
|
||||
sawFullWildcard = true
|
||||
}
|
||||
}
|
||||
for _, p := range s.Principal["AWS"] {
|
||||
if strings.HasSuffix(p, "user/reader") {
|
||||
sawReaderPrincipal = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !sawFullWildcard {
|
||||
t.Fatal("full grant did not produce an s3:* action")
|
||||
}
|
||||
if !sawReaderPrincipal {
|
||||
t.Fatal("reader principal ARN missing")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// UserKey is an S3 access/secret key pair belonging to an RGW user.
|
||||
type UserKey struct {
|
||||
User string `json:"user"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
}
|
||||
|
||||
// User is the subset of an RGW user record the operator consumes.
|
||||
type User struct {
|
||||
UID string `json:"user_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Email string `json:"email"`
|
||||
MaxBuckets int `json:"max_buckets"`
|
||||
Suspended int `json:"suspended"`
|
||||
Keys []UserKey `json:"keys"`
|
||||
}
|
||||
|
||||
// S3Key returns the first access/secret key pair, if any.
|
||||
func (u *User) S3Key() (UserKey, bool) {
|
||||
if len(u.Keys) == 0 {
|
||||
return UserKey{}, false
|
||||
}
|
||||
return u.Keys[0], true
|
||||
}
|
||||
|
||||
// UserSpec describes the desired state of an RGW user.
|
||||
type UserSpec struct {
|
||||
UID string
|
||||
DisplayName string
|
||||
Email string
|
||||
MaxBuckets *int32
|
||||
Suspended bool
|
||||
}
|
||||
|
||||
type createUserRequest struct {
|
||||
UID string `json:"uid"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Email string `json:"email,omitempty"`
|
||||
MaxBuckets *int32 `json:"max_buckets,omitempty"`
|
||||
Suspended bool `json:"suspended"`
|
||||
GenerateKey bool `json:"generate_key"`
|
||||
}
|
||||
|
||||
type updateUserRequest struct {
|
||||
DisplayName string `json:"display_name"`
|
||||
Email string `json:"email,omitempty"`
|
||||
MaxBuckets *int32 `json:"max_buckets,omitempty"`
|
||||
Suspended bool `json:"suspended"`
|
||||
}
|
||||
|
||||
// GetUser fetches an RGW user by uid, returning an *APIError with status 404
|
||||
// (see IsNotFound) when it does not exist.
|
||||
func (c *Client) GetUser(ctx context.Context, uid string) (*User, error) {
|
||||
var u User
|
||||
if err := c.do(ctx, http.MethodGet, "/api/rgw/user/"+url.PathEscape(uid), nil, &u, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// CreateUser creates an RGW user, asking the dashboard to generate an S3 key
|
||||
// pair. The returned User carries the generated keys.
|
||||
func (c *Client) CreateUser(ctx context.Context, spec UserSpec) (*User, error) {
|
||||
req := createUserRequest{
|
||||
UID: spec.UID,
|
||||
DisplayName: firstNonEmpty(spec.DisplayName, spec.UID),
|
||||
Email: spec.Email,
|
||||
MaxBuckets: spec.MaxBuckets,
|
||||
Suspended: spec.Suspended,
|
||||
GenerateKey: true,
|
||||
}
|
||||
var u User
|
||||
if err := c.do(ctx, http.MethodPost, "/api/rgw/user", req, &u, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// UpdateUser reconciles the mutable attributes of an existing RGW user.
|
||||
func (c *Client) UpdateUser(ctx context.Context, spec UserSpec) (*User, error) {
|
||||
req := updateUserRequest{
|
||||
DisplayName: firstNonEmpty(spec.DisplayName, spec.UID),
|
||||
Email: spec.Email,
|
||||
MaxBuckets: spec.MaxBuckets,
|
||||
Suspended: spec.Suspended,
|
||||
}
|
||||
var u User
|
||||
if err := c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(spec.UID), req, &u, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// DeleteUser removes an RGW user. A 404 is treated as success.
|
||||
func (c *Client) DeleteUser(ctx context.Context, uid string) error {
|
||||
err := c.do(ctx, http.MethodDelete, "/api/rgw/user/"+url.PathEscape(uid), nil, nil, "")
|
||||
if IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type quotaRequest struct {
|
||||
QuotaType string `json:"quota_type"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxSizeKb int64 `json:"max_size_kb"`
|
||||
MaxObjects int64 `json:"max_objects"`
|
||||
}
|
||||
|
||||
// SetUserQuota applies a quota to a user. quotaType is "user" or "bucket" (the
|
||||
// latter sets the per-bucket default for buckets the user owns). A nil or
|
||||
// negative limit means unlimited for that dimension.
|
||||
func (c *Client) SetUserQuota(ctx context.Context, uid, quotaType string, enabled bool, maxSizeBytes, maxObjects *int64) error {
|
||||
req := quotaRequest{
|
||||
QuotaType: quotaType,
|
||||
Enabled: enabled,
|
||||
MaxSizeKb: bytesToKb(maxSizeBytes),
|
||||
MaxObjects: valueOr(maxObjects, -1),
|
||||
}
|
||||
return c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(uid)+"/quota", req, nil, "")
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func bytesToKb(b *int64) int64 {
|
||||
if b == nil || *b < 0 {
|
||||
return -1
|
||||
}
|
||||
return *b / 1024
|
||||
}
|
||||
|
||||
func valueOr(v *int64, fallback int64) int64 {
|
||||
if v == nil || *v < 0 {
|
||||
return fallback
|
||||
}
|
||||
return *v
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// BucketReconciler provisions RGW buckets and owns the bucket's S3 policy. It
|
||||
// aggregates every BucketAccess that targets the bucket into a single policy
|
||||
// document, so the policy stays convergent no matter the order of events.
|
||||
type BucketReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=buckets/finalizers,verbs=update
|
||||
|
||||
func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var b v1alpha1.Bucket
|
||||
if err := r.Get(ctx, req.NamespacedName, &b); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
bucketName := orDefault(b.Spec.BucketName, b.Name)
|
||||
|
||||
if !b.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&b, finalizer) {
|
||||
if !b.Spec.RetainOnDelete {
|
||||
if err := r.Ceph.DeleteBucket(ctx, bucketName, b.Spec.PurgeOnDelete); err != nil {
|
||||
return r.fail(ctx, &b, "DeleteFailed", err)
|
||||
}
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&b, finalizer)
|
||||
if err := r.Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&b, finalizer) {
|
||||
if err := r.Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the owning user.
|
||||
var owner v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Spec.OwnerRef}, &owner); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.pending(ctx, &b, "OwnerMissing", fmt.Sprintf("waiting for ObjectStoreUser %q", b.Spec.OwnerRef))
|
||||
}
|
||||
return r.fail(ctx, &b, "OwnerLookupFailed", err)
|
||||
}
|
||||
if owner.Status.UID == "" || owner.Status.Phase != "Ready" {
|
||||
return r.pending(ctx, &b, "OwnerNotReady", fmt.Sprintf("ObjectStoreUser %q not ready", b.Spec.OwnerRef))
|
||||
}
|
||||
ownerUID := owner.Status.UID
|
||||
|
||||
// Ensure the bucket exists.
|
||||
info, err := r.Ceph.GetBucket(ctx, bucketName)
|
||||
if ceph.IsNotFound(err) {
|
||||
createSpec := ceph.CreateBucketSpec{
|
||||
Bucket: bucketName,
|
||||
OwnerUID: ownerUID,
|
||||
Zonegroup: b.Spec.Zonegroup,
|
||||
PlacementTarget: b.Spec.PlacementTarget,
|
||||
}
|
||||
if ol := b.Spec.ObjectLock; ol != nil && ol.Enabled {
|
||||
createSpec.LockEnabled = true
|
||||
createSpec.LockMode = string(ol.Mode)
|
||||
createSpec.LockDays = ol.Days
|
||||
createSpec.LockYears = ol.Years
|
||||
}
|
||||
info, err = r.Ceph.CreateBucket(ctx, createSpec)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "CreateFailed", err)
|
||||
}
|
||||
logger.Info("created bucket", "bucket", bucketName, "owner", ownerUID)
|
||||
} else if err != nil {
|
||||
return r.fail(ctx, &b, "LookupFailed", err)
|
||||
}
|
||||
bucketID := info.InstanceID()
|
||||
|
||||
// Versioning (forced on when object lock is enabled).
|
||||
if b.Spec.Versioning || (b.Spec.ObjectLock != nil && b.Spec.ObjectLock.Enabled) {
|
||||
if err := r.Ceph.SetBucketVersioning(ctx, bucketName, bucketID, ownerUID, true); err != nil {
|
||||
return r.fail(ctx, &b, "VersioningFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tags.
|
||||
if len(b.Spec.Tags) > 0 {
|
||||
tj, err := ceph.BuildTagJSON(b.Spec.Tags)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "TagsFailed", err)
|
||||
}
|
||||
if tj != "" {
|
||||
if err := r.Ceph.SetBucketTags(ctx, bucketName, bucketID, ownerUID, tj); err != nil {
|
||||
return r.fail(ctx, &b, "TagsFailed", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bucket default quota (applied to the owner).
|
||||
if q := b.Spec.Quota; q != nil {
|
||||
if err := r.Ceph.SetUserQuota(ctx, ownerUID, "bucket", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil {
|
||||
return r.fail(ctx, &b, "QuotaFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Render and apply the aggregate S3 policy from all BucketAccess grants.
|
||||
grants, principals, err := r.collectGrants(ctx, b.Namespace, b.Name)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "GrantsFailed", err)
|
||||
}
|
||||
policy, err := ceph.BuildBucketPolicy(bucketName, grants)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &b, "PolicyBuildFailed", err)
|
||||
}
|
||||
if err := r.Ceph.SetBucketPolicy(ctx, bucketName, bucketID, ownerUID, policy); err != nil {
|
||||
return r.fail(ctx, &b, "PolicyFailed", err)
|
||||
}
|
||||
|
||||
b.Status.Phase = "Ready"
|
||||
b.Status.BucketName = bucketName
|
||||
b.Status.BucketID = bucketID
|
||||
b.Status.Owner = ownerUID
|
||||
b.Status.PolicyPrincipals = int32(principals)
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, true, "Provisioned", "bucket provisioned")
|
||||
if err := r.Status().Update(ctx, &b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
// collectGrants returns the deduplicated set of grants for a bucket, drawn from
|
||||
// every ready, non-deleting BucketAccess that references it, plus the count of
|
||||
// distinct principals.
|
||||
func (r *BucketReconciler) collectGrants(ctx context.Context, namespace, bucketRefName string) ([]ceph.Grant, int, error) {
|
||||
var list v1alpha1.BucketAccessList
|
||||
if err := r.List(ctx, &list, client.InNamespace(namespace)); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
principals := map[string]struct{}{}
|
||||
var grants []ceph.Grant
|
||||
for i := range list.Items {
|
||||
ba := &list.Items[i]
|
||||
if ba.Spec.BucketRef != bucketRefName {
|
||||
continue
|
||||
}
|
||||
if !ba.DeletionTimestamp.IsZero() {
|
||||
continue
|
||||
}
|
||||
if ba.Status.UID == "" {
|
||||
continue
|
||||
}
|
||||
key := ba.Status.UID + "|" + string(ba.Spec.Level)
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
principals[ba.Status.UID] = struct{}{}
|
||||
grants = append(grants, ceph.Grant{UID: ba.Status.UID, Level: string(ba.Spec.Level)})
|
||||
}
|
||||
return grants, len(principals), nil
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) pending(ctx context.Context, b *v1alpha1.Bucket, reason, msg string) (ctrl.Result, error) {
|
||||
b.Status.Phase = "Pending"
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, false, reason, msg)
|
||||
if err := r.Status().Update(ctx, b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) fail(ctx context.Context, b *v1alpha1.Bucket, reason string, cause error) (ctrl.Result, error) {
|
||||
b.Status.Phase = "Error"
|
||||
b.Status.ObservedGeneration = b.Generation
|
||||
setReady(&b.Status.Conditions, b.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, b); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *BucketReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.Bucket{}).
|
||||
Watches(&v1alpha1.BucketAccess{}, handler.EnqueueRequestsFromMapFunc(r.bucketForAccess)).
|
||||
Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.bucketsForOwner)).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// bucketForAccess maps a BucketAccess change to its referenced Bucket.
|
||||
func (r *BucketReconciler) bucketForAccess(_ context.Context, obj client.Object) []reconcile.Request {
|
||||
ba, ok := obj.(*v1alpha1.BucketAccess)
|
||||
if !ok || ba.Spec.BucketRef == "" {
|
||||
return nil
|
||||
}
|
||||
return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}}}
|
||||
}
|
||||
|
||||
// bucketsForOwner maps an ObjectStoreUser change to every Bucket it owns.
|
||||
func (r *BucketReconciler) bucketsForOwner(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
osu, ok := obj.(*v1alpha1.ObjectStoreUser)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var list v1alpha1.BucketList
|
||||
if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil {
|
||||
return nil
|
||||
}
|
||||
var reqs []reconcile.Request
|
||||
for i := range list.Items {
|
||||
if list.Items[i].Spec.OwnerRef == osu.Name {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{
|
||||
Namespace: list.Items[i].Namespace, Name: list.Items[i].Name,
|
||||
}})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// BucketAccessReconciler ensures the principal for a grant exists (creating a
|
||||
// dedicated RGW user when none is referenced) and delivers its keys. The bucket
|
||||
// policy itself is owned and rendered by the Bucket controller, which watches
|
||||
// BucketAccess objects.
|
||||
type BucketAccessReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/finalizers,verbs=update
|
||||
|
||||
func (r *BucketAccessReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var ba v1alpha1.BucketAccess
|
||||
if err := r.Get(ctx, req.NamespacedName, &ba); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
managed := ba.Spec.UserRef == ""
|
||||
uid, err := r.resolveUID(ctx, &ba)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &ba, "ResolveFailed", err)
|
||||
}
|
||||
|
||||
if !ba.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&ba, finalizer) {
|
||||
// Only delete a user the operator created for this grant.
|
||||
if managed && uid != "" {
|
||||
if err := r.Ceph.DeleteUser(ctx, uid); err != nil {
|
||||
return r.fail(ctx, &ba, "DeleteFailed", err)
|
||||
}
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&ba, finalizer)
|
||||
if err := r.Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&ba, finalizer) {
|
||||
if err := r.Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the referenced Bucket so we can label the credential Secret and
|
||||
// gate the grant on the bucket existing.
|
||||
var bucket v1alpha1.Bucket
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.BucketRef}, &bucket); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.pending(ctx, &ba, "BucketMissing", fmt.Sprintf("waiting for Bucket %q", ba.Spec.BucketRef))
|
||||
}
|
||||
return r.fail(ctx, &ba, "BucketLookupFailed", err)
|
||||
}
|
||||
bucketName := orDefault(bucket.Status.BucketName, orDefault(bucket.Spec.BucketName, bucket.Name))
|
||||
|
||||
secretName := ba.Status.SecretName
|
||||
if managed {
|
||||
if uid == "" {
|
||||
uid = fmt.Sprintf("%s-%s", ba.Spec.BucketRef, ba.Name)
|
||||
}
|
||||
secretName = orDefault(ba.Spec.SecretName, ba.Name+"-rgw")
|
||||
|
||||
user, err := r.ensureUser(ctx, uid)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &ba, "UserFailed", err)
|
||||
}
|
||||
key, ok := user.S3Key()
|
||||
if !ok {
|
||||
return r.fail(ctx, &ba, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid))
|
||||
}
|
||||
if err := upsertSecret(ctx, r.Client, r.Scheme, &ba, secretName, ba.Namespace,
|
||||
credentialSecretData(key, uid, r.Endpoint, bucketName)); err != nil {
|
||||
return r.fail(ctx, &ba, "SecretFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
ba.Status.Phase = "Ready"
|
||||
ba.Status.UID = uid
|
||||
ba.Status.SecretName = secretName
|
||||
ba.Status.Bound = true
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, true, "Granted",
|
||||
fmt.Sprintf("%s access for %s applied to bucket %s", ba.Spec.Level, uid, bucketName))
|
||||
if err := r.Status().Update(ctx, &ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
logger.Info("bucket access reconciled", "bucket", bucketName, "uid", uid, "level", ba.Spec.Level)
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
// resolveUID returns the RGW uid this grant targets: the referenced
|
||||
// ObjectStoreUser's provisioned uid, or the managed uid derived from the spec.
|
||||
func (r *BucketAccessReconciler) resolveUID(ctx context.Context, ba *v1alpha1.BucketAccess) (string, error) {
|
||||
if ba.Spec.UserRef == "" {
|
||||
if ba.Spec.UID != "" {
|
||||
return ba.Spec.UID, nil
|
||||
}
|
||||
// Derived lazily in Reconcile once we know it is not a deletion no-op.
|
||||
return "", nil
|
||||
}
|
||||
var osu v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: ba.Namespace, Name: ba.Spec.UserRef}, &osu); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if osu.Status.UID == "" {
|
||||
return "", fmt.Errorf("ObjectStoreUser %q not ready", ba.Spec.UserRef)
|
||||
}
|
||||
return osu.Status.UID, nil
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) ensureUser(ctx context.Context, uid string) (*ceph.User, error) {
|
||||
if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) {
|
||||
if _, err := r.Ceph.CreateUser(ctx, ceph.UserSpec{UID: uid, DisplayName: uid}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Ceph.GetUser(ctx, uid)
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) pending(ctx context.Context, ba *v1alpha1.BucketAccess, reason, msg string) (ctrl.Result, error) {
|
||||
ba.Status.Phase = "Pending"
|
||||
ba.Status.Bound = false
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, false, reason, msg)
|
||||
if err := r.Status().Update(ctx, ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueShort}, nil
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) fail(ctx context.Context, ba *v1alpha1.BucketAccess, reason string, cause error) (ctrl.Result, error) {
|
||||
ba.Status.Phase = "Error"
|
||||
ba.Status.Bound = false
|
||||
ba.Status.ObservedGeneration = ba.Generation
|
||||
setReady(&ba.Status.Conditions, ba.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, ba); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *BucketAccessReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.BucketAccess{}).
|
||||
Watches(&v1alpha1.ObjectStoreUser{}, handler.EnqueueRequestsFromMapFunc(r.accessForUser)).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// accessForUser maps an ObjectStoreUser change to every BucketAccess that
|
||||
// references it, so a grant binds as soon as its user becomes ready.
|
||||
func (r *BucketAccessReconciler) accessForUser(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
osu, ok := obj.(*v1alpha1.ObjectStoreUser)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var list v1alpha1.BucketAccessList
|
||||
if err := r.List(ctx, &list, client.InNamespace(osu.Namespace)); err != nil {
|
||||
return nil
|
||||
}
|
||||
var reqs []reconcile.Request
|
||||
for i := range list.Items {
|
||||
if list.Items[i].Spec.UserRef == osu.Name {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{
|
||||
Namespace: list.Items[i].Namespace, Name: list.Items[i].Name,
|
||||
}})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// finalizer guards external RGW state (users, buckets, policy statements) so it
|
||||
// is cleaned up before the Kubernetes object disappears.
|
||||
const finalizer = "ceph.unkin.net/finalizer"
|
||||
|
||||
// requeueSteady is the resync interval for healthy objects; it lets the
|
||||
// operator heal drift made directly against RGW.
|
||||
const requeueSteady = 10 * time.Minute
|
||||
|
||||
// requeueShort backs off on transient "waiting for a dependency" states.
|
||||
const requeueShort = 30 * time.Second
|
||||
|
||||
// setReady sets the standard Ready condition on a status conditions slice.
|
||||
func setReady(conds *[]metav1.Condition, gen int64, ok bool, reason, msg string) {
|
||||
status := metav1.ConditionFalse
|
||||
if ok {
|
||||
status = metav1.ConditionTrue
|
||||
}
|
||||
meta.SetStatusCondition(conds, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: status,
|
||||
ObservedGeneration: gen,
|
||||
Reason: reason,
|
||||
Message: truncate(msg, 32000),
|
||||
})
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func orDefault(v, def string) string {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// upsertSecret creates or updates an owner-referenced Opaque Secret with data.
|
||||
func upsertSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, name, namespace string, data map[string][]byte) error {
|
||||
sec := &corev1.Secret{}
|
||||
sec.Name = name
|
||||
sec.Namespace = namespace
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, c, sec, func() error {
|
||||
sec.Type = corev1.SecretTypeOpaque
|
||||
if sec.Data == nil {
|
||||
sec.Data = map[string][]byte{}
|
||||
}
|
||||
for k, v := range data {
|
||||
sec.Data[k] = v
|
||||
}
|
||||
return controllerutil.SetControllerReference(owner, sec, scheme)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// credentialSecretData assembles the conventional S3/AWS credential keys.
|
||||
func credentialSecretData(key ceph.UserKey, uid, endpoint, bucket string) map[string][]byte {
|
||||
data := map[string][]byte{
|
||||
"AWS_ACCESS_KEY_ID": []byte(key.AccessKey),
|
||||
"AWS_SECRET_ACCESS_KEY": []byte(key.SecretKey),
|
||||
"RGW_UID": []byte(uid),
|
||||
}
|
||||
if endpoint != "" {
|
||||
data["S3_ENDPOINT"] = []byte(endpoint)
|
||||
if host := hostOf(endpoint); host != "" {
|
||||
data["BUCKET_HOST"] = []byte(host)
|
||||
}
|
||||
}
|
||||
if bucket != "" {
|
||||
data["BUCKET_NAME"] = []byte(bucket)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func hostOf(endpoint string) string {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil || u.Host == "" {
|
||||
return endpoint
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// ObjectStoreUserReconciler provisions RGW users and delivers their keys.
|
||||
type ObjectStoreUserReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Ceph *ceph.Client
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
|
||||
|
||||
func (r *ObjectStoreUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
var osu v1alpha1.ObjectStoreUser
|
||||
if err := r.Get(ctx, req.NamespacedName, &osu); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
uid := orDefault(osu.Spec.UID, osu.Name)
|
||||
secretName := orDefault(osu.Spec.SecretName, osu.Name+"-rgw")
|
||||
|
||||
if !osu.DeletionTimestamp.IsZero() {
|
||||
if controllerutil.ContainsFinalizer(&osu, finalizer) {
|
||||
if err := r.Ceph.DeleteUser(ctx, uid); err != nil {
|
||||
return r.fail(ctx, &osu, "DeleteFailed", err)
|
||||
}
|
||||
controllerutil.RemoveFinalizer(&osu, finalizer)
|
||||
if err := r.Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if controllerutil.AddFinalizer(&osu, finalizer) {
|
||||
if err := r.Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
spec := ceph.UserSpec{
|
||||
UID: uid,
|
||||
DisplayName: osu.Spec.DisplayName,
|
||||
Email: osu.Spec.Email,
|
||||
MaxBuckets: osu.Spec.MaxBuckets,
|
||||
Suspended: osu.Spec.Suspended,
|
||||
}
|
||||
|
||||
if _, err := r.Ceph.GetUser(ctx, uid); ceph.IsNotFound(err) {
|
||||
if _, err := r.Ceph.CreateUser(ctx, spec); err != nil {
|
||||
return r.fail(ctx, &osu, "CreateFailed", err)
|
||||
}
|
||||
logger.Info("created RGW user", "uid", uid)
|
||||
} else if err != nil {
|
||||
return r.fail(ctx, &osu, "LookupFailed", err)
|
||||
} else {
|
||||
if _, err := r.Ceph.UpdateUser(ctx, spec); err != nil {
|
||||
return r.fail(ctx, &osu, "UpdateFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
if q := osu.Spec.Quota; q != nil {
|
||||
if err := r.Ceph.SetUserQuota(ctx, uid, "user", q.Enabled, q.MaxSizeBytes, q.MaxObjects); err != nil {
|
||||
return r.fail(ctx, &osu, "QuotaFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
user, err := r.Ceph.GetUser(ctx, uid)
|
||||
if err != nil {
|
||||
return r.fail(ctx, &osu, "LookupFailed", err)
|
||||
}
|
||||
key, ok := user.S3Key()
|
||||
if !ok {
|
||||
return r.fail(ctx, &osu, "NoKeys", fmt.Errorf("user %s has no S3 keys", uid))
|
||||
}
|
||||
|
||||
if err := upsertSecret(ctx, r.Client, r.Scheme, &osu, secretName, osu.Namespace,
|
||||
credentialSecretData(key, uid, r.Endpoint, "")); err != nil {
|
||||
return r.fail(ctx, &osu, "SecretFailed", err)
|
||||
}
|
||||
|
||||
osu.Status.Phase = "Ready"
|
||||
osu.Status.UID = uid
|
||||
osu.Status.SecretName = secretName
|
||||
osu.Status.ObservedGeneration = osu.Generation
|
||||
setReady(&osu.Status.Conditions, osu.Generation, true, "Provisioned", "RGW user provisioned")
|
||||
if err := r.Status().Update(ctx, &osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: requeueSteady}, nil
|
||||
}
|
||||
|
||||
func (r *ObjectStoreUserReconciler) fail(ctx context.Context, osu *v1alpha1.ObjectStoreUser, reason string, cause error) (ctrl.Result, error) {
|
||||
osu.Status.Phase = "Error"
|
||||
osu.Status.ObservedGeneration = osu.Generation
|
||||
setReady(&osu.Status.Conditions, osu.Generation, false, reason, cause.Error())
|
||||
if err := r.Status().Update(ctx, osu); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, cause
|
||||
}
|
||||
|
||||
func (r *ObjectStoreUserReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1alpha1.ObjectStoreUser{}).
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
"git.unkin.net/unkin/cephrgw-operator/internal/ceph"
|
||||
)
|
||||
|
||||
// SetupAll registers every controller with the manager.
|
||||
func SetupAll(mgr ctrl.Manager, cephClient *ceph.Client, endpoint string) error {
|
||||
if err := (&ObjectStoreUserReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := (&BucketReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := (&BucketAccessReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Ceph: cephClient,
|
||||
Endpoint: endpoint,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user