From 8ca166a49414c64161ee2978b6310e4469eac5f7 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 15 Nov 2026 19:52:08 +1100 Subject: [PATCH 1/3] ceph-system: run radosgw as a deployment --- apps/base/ceph-system/configmap.yaml | 25 ++++ apps/base/ceph-system/deployment.yaml | 124 ++++++++++++++++++ apps/base/ceph-system/kustomization.yaml | 12 ++ apps/base/ceph-system/namespace.yaml | 5 + apps/base/ceph-system/pdb.yaml | 11 ++ apps/base/ceph-system/service.yaml | 16 +++ apps/base/ceph-system/vaultauth.yaml | 18 +++ apps/base/ceph-system/vaultstaticsecret.yaml | 17 +++ .../au-syd1/ceph-system/kustomization.yaml | 6 + argocd/applicationsets/platform.yaml | 1 + 10 files changed, 235 insertions(+) create mode 100644 apps/base/ceph-system/configmap.yaml create mode 100644 apps/base/ceph-system/deployment.yaml create mode 100644 apps/base/ceph-system/kustomization.yaml create mode 100644 apps/base/ceph-system/namespace.yaml create mode 100644 apps/base/ceph-system/pdb.yaml create mode 100644 apps/base/ceph-system/service.yaml create mode 100644 apps/base/ceph-system/vaultauth.yaml create mode 100644 apps/base/ceph-system/vaultstaticsecret.yaml create mode 100644 apps/overlays/au-syd1/ceph-system/kustomization.yaml diff --git a/apps/base/ceph-system/configmap.yaml b/apps/base/ceph-system/configmap.yaml new file mode 100644 index 0000000..e6047e3 --- /dev/null +++ b/apps/base/ceph-system/configmap.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-conf + namespace: ceph-system +data: + ceph.conf: | + [global] + fsid = de96a98f-3d23-465a-a899-86d3d67edab8 + mon_host = 198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13 + auth_client_required = cephx + auth_cluster_required = cephx + auth_service_required = cephx + ms_bind_ipv4 = true + ms_bind_ipv6 = false + + [client.rgw.syd1] + rgw_realm = unkin + rgw_zonegroup = au + rgw_zone = syd1 + rgw_frontends = beast port=7480 + log_to_file = false + log_to_stderr = true + err_to_stderr = true diff --git a/apps/base/ceph-system/deployment.yaml b/apps/base/ceph-system/deployment.yaml new file mode 100644 index 0000000..d1fe170 --- /dev/null +++ b/apps/base/ceph-system/deployment.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rgw + namespace: ceph-system + labels: + app.kubernetes.io/name: rgw + app.kubernetes.io/component: rgw + app.kubernetes.io/version: v18.2.8 + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app.kubernetes.io/name: rgw + template: + metadata: + labels: + app.kubernetes.io/name: rgw + app.kubernetes.io/component: rgw + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 167 + runAsGroup: 167 + fsGroup: 167 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: rgw + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: rgw + containers: + - name: rgw + image: quay.io/ceph/ceph:v18.2.8 + imagePullPolicy: IfNotPresent + command: + - radosgw + - -f + - --cluster + - ceph + - --name + - client.rgw.syd1 + - --setuser + - ceph + - --setgroup + - ceph + ports: + - name: http + containerPort: 7480 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + readOnly: true + - name: keyring + mountPath: /var/lib/ceph/radosgw/ceph-rgw.syd1 + readOnly: true + - name: run + mountPath: /var/run/ceph + - name: log + mountPath: /var/log/ceph + - name: tmp + mountPath: /tmp + volumes: + - name: ceph-conf + configMap: + name: ceph-conf + - name: keyring + secret: + secretName: rgw-keyring + items: + - key: keyring + path: keyring + - name: run + emptyDir: {} + - name: log + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/apps/base/ceph-system/kustomization.yaml b/apps/base/ceph-system/kustomization.yaml new file mode 100644 index 0000000..18bd7ff --- /dev/null +++ b/apps/base/ceph-system/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - deployment.yaml + - service.yaml + - pdb.yaml diff --git a/apps/base/ceph-system/namespace.yaml b/apps/base/ceph-system/namespace.yaml new file mode 100644 index 0000000..31432d9 --- /dev/null +++ b/apps/base/ceph-system/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ceph-system diff --git a/apps/base/ceph-system/pdb.yaml b/apps/base/ceph-system/pdb.yaml new file mode 100644 index 0000000..c83eccf --- /dev/null +++ b/apps/base/ceph-system/pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rgw + namespace: ceph-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: rgw diff --git a/apps/base/ceph-system/service.yaml b/apps/base/ceph-system/service.yaml new file mode 100644 index 0000000..93d7174 --- /dev/null +++ b/apps/base/ceph-system/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: rgw + namespace: ceph-system + labels: + app.kubernetes.io/name: rgw +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: rgw + ports: + - name: http + port: 7480 + targetPort: http diff --git a/apps/base/ceph-system/vaultauth.yaml b/apps/base/ceph-system/vaultauth.yaml new file mode 100644 index 0000000..bea2fb9 --- /dev/null +++ b/apps/base/ceph-system/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: ceph-system +spec: + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default + allowedNamespaces: + - ceph-system + kubernetes: + role: default + serviceAccount: default + audiences: + - vault + tokenExpirationSeconds: 600 diff --git a/apps/base/ceph-system/vaultstaticsecret.yaml b/apps/base/ceph-system/vaultstaticsecret.yaml new file mode 100644 index 0000000..b10d6ed --- /dev/null +++ b/apps/base/ceph-system/vaultstaticsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: rgw-keyring + namespace: ceph-system +spec: + vaultAuthRef: default + mount: kv + type: kv-v2 + path: kubernetes/namespace/ceph-system/default/rgw-keyring + refreshAfter: 5m + hmacSecretData: true + destination: + name: rgw-keyring + create: true + overwrite: true diff --git a/apps/overlays/au-syd1/ceph-system/kustomization.yaml b/apps/overlays/au-syd1/ceph-system/kustomization.yaml new file mode 100644 index 0000000..eeaf1e9 --- /dev/null +++ b/apps/overlays/au-syd1/ceph-system/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/ceph-system diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index 2376860..515df7e 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -20,6 +20,7 @@ spec: - path: apps/overlays/*/cattle-system - path: apps/overlays/*/cert-manager - path: apps/overlays/*/certificates + - path: apps/overlays/*/ceph-system - path: apps/overlays/*/cephrgw-system - path: apps/overlays/*/cnpg-system - path: apps/overlays/*/consul -- 2.47.3 From a08da880a96ed0069e2d318057ca8971039b1406 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 15 Nov 2026 19:57:09 +1100 Subject: [PATCH 2/3] Require rgw host spread, add VPA, drop redundant setuser --- apps/base/ceph-system/deployment.yaml | 6 +----- apps/base/ceph-system/kustomization.yaml | 1 + apps/base/ceph-system/vaultstaticsecret.yaml | 1 + apps/base/ceph-system/vpa.yaml | 13 +++++++++++++ 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 apps/base/ceph-system/vpa.yaml diff --git a/apps/base/ceph-system/deployment.yaml b/apps/base/ceph-system/deployment.yaml index d1fe170..8f85248 100644 --- a/apps/base/ceph-system/deployment.yaml +++ b/apps/base/ceph-system/deployment.yaml @@ -46,7 +46,7 @@ spec: topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule labelSelector: matchLabels: app.kubernetes.io/name: rgw @@ -61,10 +61,6 @@ spec: - ceph - --name - client.rgw.syd1 - - --setuser - - ceph - - --setgroup - - ceph ports: - name: http containerPort: 7480 diff --git a/apps/base/ceph-system/kustomization.yaml b/apps/base/ceph-system/kustomization.yaml index 18bd7ff..e8ab320 100644 --- a/apps/base/ceph-system/kustomization.yaml +++ b/apps/base/ceph-system/kustomization.yaml @@ -10,3 +10,4 @@ resources: - deployment.yaml - service.yaml - pdb.yaml + - vpa.yaml diff --git a/apps/base/ceph-system/vaultstaticsecret.yaml b/apps/base/ceph-system/vaultstaticsecret.yaml index b10d6ed..59271b7 100644 --- a/apps/base/ceph-system/vaultstaticsecret.yaml +++ b/apps/base/ceph-system/vaultstaticsecret.yaml @@ -1,4 +1,5 @@ --- +# Seed kv/kubernetes/namespace/ceph-system/default/rgw-keyring field `keyring` with the client.rgw.syd1 keyring. apiVersion: secrets.hashicorp.com/v1beta1 kind: VaultStaticSecret metadata: diff --git a/apps/base/ceph-system/vpa.yaml b/apps/base/ceph-system/vpa.yaml new file mode 100644 index 0000000..c04938f --- /dev/null +++ b/apps/base/ceph-system/vpa.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: rgw-vpa + namespace: ceph-system +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: rgw + updatePolicy: + updateMode: "Off" -- 2.47.3 From 490388793f9556b99fcdcea06adada6743783bf2 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 15 Nov 2026 20:04:17 +1100 Subject: [PATCH 3/3] ceph-system: pin rgw keyring path, add startup probe, raise memory request --- apps/base/ceph-system/configmap.yaml | 1 + apps/base/ceph-system/deployment.yaml | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/base/ceph-system/configmap.yaml b/apps/base/ceph-system/configmap.yaml index e6047e3..52a1dff 100644 --- a/apps/base/ceph-system/configmap.yaml +++ b/apps/base/ceph-system/configmap.yaml @@ -20,6 +20,7 @@ data: rgw_zonegroup = au rgw_zone = syd1 rgw_frontends = beast port=7480 + keyring = /var/lib/ceph/radosgw/ceph-rgw.syd1/keyring log_to_file = false log_to_stderr = true err_to_stderr = true diff --git a/apps/base/ceph-system/deployment.yaml b/apps/base/ceph-system/deployment.yaml index 8f85248..32ebbd6 100644 --- a/apps/base/ceph-system/deployment.yaml +++ b/apps/base/ceph-system/deployment.yaml @@ -64,6 +64,12 @@ spec: ports: - name: http containerPort: 7480 + startupProbe: + httpGet: + path: / + port: http + periodSeconds: 5 + failureThreshold: 30 readinessProbe: httpGet: path: / @@ -74,13 +80,12 @@ spec: httpGet: path: / port: http - initialDelaySeconds: 30 periodSeconds: 20 failureThreshold: 6 resources: requests: cpu: "1" - memory: 1Gi + memory: 2Gi limits: cpu: "2" memory: 4Gi -- 2.47.3