ceph-system: run radosgw as a deployment (#583)

Part of #582. RGW runs on VMs (ausyd1nxvm2115-2119) and needs to move into k8s under GitOps; this adds the `ceph-system` namespace with an in-cluster RGW behind a ClusterIP service. Cutover comes later.

- add `ceph-system` app to the platform applicationset
- add ceph.conf configmap with a shared `client.rgw.syd1` section and explicit keyring path
- add rgw keyring from vault `kubernetes/namespace/ceph-system/default/rgw-keyring` (`keyring`)
- add 3-replica `quay.io/ceph/ceph:v18.2.8` radosgw deployment, non-root, read-only root, startup probe, hard per-node spread, PDB + VPA
- add ClusterIP service `rgw` on 7480

Requires unkin/terraform-artifactapi#78 merged first.

Reviewed-on: #583
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #583.
This commit is contained in:
2026-11-15 21:51:59 +11:00
committed by BenVincent
parent 166605d3f1
commit 26e53ee35c
11 changed files with 252 additions and 0 deletions
+26
View File
@@ -0,0 +1,26 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ceph-conf
namespace: ceph-system
data:
ceph.conf: |
[global]
fsid = de96a98f-3d23-465a-a899-86d3d67edab8
mon_host = 198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13
auth_client_required = cephx
auth_cluster_required = cephx
auth_service_required = cephx
ms_bind_ipv4 = true
ms_bind_ipv6 = false
[client.rgw.syd1]
rgw_realm = unkin
rgw_zonegroup = au
rgw_zone = syd1
rgw_frontends = beast port=7480
keyring = /var/lib/ceph/radosgw/ceph-rgw.syd1/keyring
log_to_file = false
log_to_stderr = true
err_to_stderr = true
+125
View File
@@ -0,0 +1,125 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rgw
namespace: ceph-system
labels:
app.kubernetes.io/name: rgw
app.kubernetes.io/component: rgw
app.kubernetes.io/version: v18.2.8
annotations:
reloader.stakater.com/auto: "true"
spec:
replicas: 3
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 0
selector:
matchLabels:
app.kubernetes.io/name: rgw
template:
metadata:
labels:
app.kubernetes.io/name: rgw
app.kubernetes.io/component: rgw
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 167
runAsGroup: 167
fsGroup: 167
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: rgw
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: rgw
containers:
- name: rgw
image: quay.io/ceph/ceph:v18.2.8
imagePullPolicy: IfNotPresent
command:
- radosgw
- -f
- --cluster
- ceph
- --name
- client.rgw.syd1
ports:
- name: http
containerPort: 7480
startupProbe:
httpGet:
path: /
port: http
periodSeconds: 5
failureThreshold: 30
readinessProbe:
httpGet:
path: /
port: http
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: http
periodSeconds: 20
failureThreshold: 6
resources:
requests:
cpu: "1"
memory: 2Gi
limits:
cpu: "2"
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
volumeMounts:
- name: ceph-conf
mountPath: /etc/ceph
readOnly: true
- name: keyring
mountPath: /var/lib/ceph/radosgw/ceph-rgw.syd1
readOnly: true
- name: run
mountPath: /var/run/ceph
- name: log
mountPath: /var/log/ceph
- name: tmp
mountPath: /tmp
volumes:
- name: ceph-conf
configMap:
name: ceph-conf
- name: keyring
secret:
secretName: rgw-keyring
items:
- key: keyring
path: keyring
- name: run
emptyDir: {}
- name: log
emptyDir: {}
- name: tmp
emptyDir: {}
+13
View File
@@ -0,0 +1,13 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- configmap.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- deployment.yaml
- service.yaml
- pdb.yaml
- vpa.yaml
+5
View File
@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: ceph-system
+11
View File
@@ -0,0 +1,11 @@
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: rgw
namespace: ceph-system
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: rgw
+16
View File
@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
name: rgw
namespace: ceph-system
labels:
app.kubernetes.io/name: rgw
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: rgw
ports:
- name: http
port: 7480
targetPort: http
+18
View File
@@ -0,0 +1,18 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: ceph-system
spec:
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
allowedNamespaces:
- ceph-system
kubernetes:
role: default
serviceAccount: default
audiences:
- vault
tokenExpirationSeconds: 600
@@ -0,0 +1,18 @@
---
# Seed kv/kubernetes/namespace/ceph-system/default/rgw-keyring field `keyring` with the client.rgw.syd1 keyring.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: rgw-keyring
namespace: ceph-system
spec:
vaultAuthRef: default
mount: kv
type: kv-v2
path: kubernetes/namespace/ceph-system/default/rgw-keyring
refreshAfter: 5m
hmacSecretData: true
destination:
name: rgw-keyring
create: true
overwrite: true
+13
View File
@@ -0,0 +1,13 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: rgw-vpa
namespace: ceph-system
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: rgw
updatePolicy:
updateMode: "Off"