Talk to radosgw directly via go-ceph + aws-sdk-go-v2

The operator drove the Ceph manager dashboard REST API to manage RGW users,
buckets and policies. That coupled it to a dashboard login, the dashboard's RGW
wiring, and the dashboard's bucket API surface. Rebuild the Ceph integration to
talk directly to radosgw the way the CLI does, using native Go libraries, while
keeping every operator capability identical.

The exported surface of internal/ceph is unchanged, so the three controllers
and cmd/operator's structure are untouched (bar the CEPH_RGW_* config plumbing).

- replace the internal/ceph client internals with github.com/ceph/go-ceph
  rgw/admin (Admin Ops API) for users, keys, quotas and bucket info/removal
- add github.com/aws/aws-sdk-go-v2 S3 client for bucket create, versioning,
  policy, tagging and object lock, signed as the bucket owner
- map go-ceph admin.ErrNoSuch*/ErrUserExists and smithy APIError codes into
  IsNotFound/IsConflict so controller create-vs-update branching is preserved
- set S3 path-style addressing and WhenRequired checksum modes for RGW
- delete the hand-rolled dashboard client, token auth and JSON plumbing
- keep policy.go/BuildBucketPolicy/BuildTagJSON as pure builders
- replace the client tests with NewClient validation and error-classifier tests
- keep CGO_ENABLED=0 distroless: only go-ceph's pure-Go rgw/admin is imported
- switch env/config to CEPH_RGW_* (endpoint, admin endpoint, access/secret key,
  region, CA, insecure) and update the deployment manifest
- rewrite README and docs/ceph-setup.md for the single RGW admin user
  (caps users=*;buckets=*), keeping Vault/VSO as the primary credential source

Claude-Session: https://claude.ai/code/session_016CEncETbf8cvy1PhsHfFHM

internal/ceph/users.go

@@ -2,25 +2,25 @@ package ceph
 
 import (
 	"context"
-	"net/http"
-	"net/url"
+
+	"github.com/ceph/go-ceph/rgw/admin"
 )
 
 // UserKey is an S3 access/secret key pair belonging to an RGW user.
 type UserKey struct {
-	User      string `json:"user"`
-	AccessKey string `json:"access_key"`
-	SecretKey string `json:"secret_key"`
+	User      string
+	AccessKey string
+	SecretKey string
 }
 
 // User is the subset of an RGW user record the operator consumes.
 type User struct {
-	UID         string    `json:"user_id"`
-	DisplayName string    `json:"display_name"`
-	Email       string    `json:"email"`
-	MaxBuckets  int       `json:"max_buckets"`
-	Suspended   int       `json:"suspended"`
-	Keys        []UserKey `json:"keys"`
+	UID         string
+	DisplayName string
+	Email       string
+	MaxBuckets  int
+	Suspended   int
+	Keys        []UserKey
 }
 
 // S3Key returns the first access/secret key pair, if any.
@@ -40,92 +40,87 @@ type UserSpec struct {
 	Suspended   bool
 }
 
-type createUserRequest struct {
-	UID         string `json:"uid"`
-	DisplayName string `json:"display_name"`
-	Email       string `json:"email,omitempty"`
-	MaxBuckets  *int32 `json:"max_buckets,omitempty"`
-	Suspended   bool   `json:"suspended"`
-	GenerateKey bool   `json:"generate_key"`
+// fromAdminUser converts a go-ceph admin.User into the subset the operator uses.
+func fromAdminUser(u admin.User) *User {
+	out := &User{
+		UID:         u.ID,
+		DisplayName: u.DisplayName,
+		Email:       u.Email,
+		MaxBuckets:  derefInt(u.MaxBuckets),
+		Suspended:   derefInt(u.Suspended),
+	}
+	for _, k := range u.Keys {
+		out.Keys = append(out.Keys, UserKey{User: k.User, AccessKey: k.AccessKey, SecretKey: k.SecretKey})
+	}
+	return out
 }
 
-type updateUserRequest struct {
-	DisplayName string `json:"display_name"`
-	Email       string `json:"email,omitempty"`
-	MaxBuckets  *int32 `json:"max_buckets,omitempty"`
-	Suspended   bool   `json:"suspended"`
-}
-
-// GetUser fetches an RGW user by uid, returning an *APIError with status 404
-// (see IsNotFound) when it does not exist.
+// GetUser fetches an RGW user by uid, returning an error classified by
+// IsNotFound (admin.ErrNoSuchUser) when it does not exist.
 func (c *Client) GetUser(ctx context.Context, uid string) (*User, error) {
-	var u User
-	if err := c.do(ctx, http.MethodGet, "/api/rgw/user/"+url.PathEscape(uid), nil, &u, ""); err != nil {
+	u, err := c.admin.GetUser(ctx, admin.User{ID: uid})
+	if err != nil {
 		return nil, err
 	}
-	return &u, nil
+	return fromAdminUser(u), nil
 }
 
-// CreateUser creates an RGW user, asking the dashboard to generate an S3 key
-// pair. The returned User carries the generated keys.
+// CreateUser creates an RGW user, asking radosgw to generate an S3 key pair. The
+// returned User carries the generated keys.
 func (c *Client) CreateUser(ctx context.Context, spec UserSpec) (*User, error) {
-	req := createUserRequest{
-		UID:         spec.UID,
+	u, err := c.admin.CreateUser(ctx, admin.User{
+		ID:          spec.UID,
 		DisplayName: firstNonEmpty(spec.DisplayName, spec.UID),
 		Email:       spec.Email,
-		MaxBuckets:  spec.MaxBuckets,
-		Suspended:   spec.Suspended,
-		GenerateKey: true,
-	}
-	var u User
-	if err := c.do(ctx, http.MethodPost, "/api/rgw/user", req, &u, ""); err != nil {
+		MaxBuckets:  int32PtrToIntPtr(spec.MaxBuckets),
+		Suspended:   boolToIntPtr(spec.Suspended),
+		GenerateKey: boolPtr(true),
+	})
+	if err != nil {
 		return nil, err
 	}
-	return &u, nil
+	c.forgetIdentity(spec.UID)
+	return fromAdminUser(u), nil
 }
 
 // UpdateUser reconciles the mutable attributes of an existing RGW user.
 func (c *Client) UpdateUser(ctx context.Context, spec UserSpec) (*User, error) {
-	req := updateUserRequest{
+	u, err := c.admin.ModifyUser(ctx, admin.User{
+		ID:          spec.UID,
 		DisplayName: firstNonEmpty(spec.DisplayName, spec.UID),
 		Email:       spec.Email,
-		MaxBuckets:  spec.MaxBuckets,
-		Suspended:   spec.Suspended,
-	}
-	var u User
-	if err := c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(spec.UID), req, &u, ""); err != nil {
+		MaxBuckets:  int32PtrToIntPtr(spec.MaxBuckets),
+		Suspended:   boolToIntPtr(spec.Suspended),
+	})
+	if err != nil {
 		return nil, err
 	}
-	return &u, nil
+	return fromAdminUser(u), nil
 }
 
-// DeleteUser removes an RGW user. A 404 is treated as success.
+// DeleteUser removes an RGW user. A NoSuchUser response is treated as success.
 func (c *Client) DeleteUser(ctx context.Context, uid string) error {
-	err := c.do(ctx, http.MethodDelete, "/api/rgw/user/"+url.PathEscape(uid), nil, nil, "")
+	err := c.admin.RemoveUser(ctx, admin.User{ID: uid})
+	c.forgetIdentity(uid)
 	if IsNotFound(err) {
 		return nil
 	}
 	return err
 }
 
-type quotaRequest struct {
-	QuotaType  string `json:"quota_type"`
-	Enabled    bool   `json:"enabled"`
-	MaxSizeKb  int64  `json:"max_size_kb"`
-	MaxObjects int64  `json:"max_objects"`
-}
-
 // SetUserQuota applies a quota to a user. quotaType is "user" or "bucket" (the
 // latter sets the per-bucket default for buckets the user owns). A nil or
 // negative limit means unlimited for that dimension.
 func (c *Client) SetUserQuota(ctx context.Context, uid, quotaType string, enabled bool, maxSizeBytes, maxObjects *int64) error {
-	req := quotaRequest{
+	maxSize := valueOr(maxSizeBytes, -1)
+	maxObj := valueOr(maxObjects, -1)
+	return c.admin.SetUserQuota(ctx, admin.QuotaSpec{
+		UID:        uid,
 		QuotaType:  quotaType,
-		Enabled:    enabled,
-		MaxSizeKb:  bytesToKb(maxSizeBytes),
-		MaxObjects: valueOr(maxObjects, -1),
-	}
-	return c.do(ctx, http.MethodPut, "/api/rgw/user/"+url.PathEscape(uid)+"/quota", req, nil, "")
+		Enabled:    &enabled,
+		MaxSize:    &maxSize,
+		MaxObjects: &maxObj,
+	})
 }
 
 func firstNonEmpty(vals ...string) string {
@@ -137,16 +132,34 @@ func firstNonEmpty(vals ...string) string {
 	return ""
 }
 
-func bytesToKb(b *int64) int64 {
-	if b == nil || *b < 0 {
-		return -1
-	}
-	return *b / 1024
-}
-
 func valueOr(v *int64, fallback int64) int64 {
 	if v == nil || *v < 0 {
 		return fallback
 	}
 	return *v
 }
+
+func derefInt(p *int) int {
+	if p == nil {
+		return 0
+	}
+	return *p
+}
+
+func boolPtr(b bool) *bool { return &b }
+
+func int32PtrToIntPtr(p *int32) *int {
+	if p == nil {
+		return nil
+	}
+	v := int(*p)
+	return &v
+}
+
+func boolToIntPtr(b bool) *int {
+	v := 0
+	if b {
+		v = 1
+	}
+	return &v
+}