From 8ca166a49414c64161ee2978b6310e4469eac5f7 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 15 Nov 2026 19:52:08 +1100 Subject: [PATCH] ceph-system: run radosgw as a deployment --- apps/base/ceph-system/configmap.yaml | 25 ++++ apps/base/ceph-system/deployment.yaml | 124 ++++++++++++++++++ apps/base/ceph-system/kustomization.yaml | 12 ++ apps/base/ceph-system/namespace.yaml | 5 + apps/base/ceph-system/pdb.yaml | 11 ++ apps/base/ceph-system/service.yaml | 16 +++ apps/base/ceph-system/vaultauth.yaml | 18 +++ apps/base/ceph-system/vaultstaticsecret.yaml | 17 +++ .../au-syd1/ceph-system/kustomization.yaml | 6 + argocd/applicationsets/platform.yaml | 1 + 10 files changed, 235 insertions(+) create mode 100644 apps/base/ceph-system/configmap.yaml create mode 100644 apps/base/ceph-system/deployment.yaml create mode 100644 apps/base/ceph-system/kustomization.yaml create mode 100644 apps/base/ceph-system/namespace.yaml create mode 100644 apps/base/ceph-system/pdb.yaml create mode 100644 apps/base/ceph-system/service.yaml create mode 100644 apps/base/ceph-system/vaultauth.yaml create mode 100644 apps/base/ceph-system/vaultstaticsecret.yaml create mode 100644 apps/overlays/au-syd1/ceph-system/kustomization.yaml diff --git a/apps/base/ceph-system/configmap.yaml b/apps/base/ceph-system/configmap.yaml new file mode 100644 index 0000000..e6047e3 --- /dev/null +++ b/apps/base/ceph-system/configmap.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-conf + namespace: ceph-system +data: + ceph.conf: | + [global] + fsid = de96a98f-3d23-465a-a899-86d3d67edab8 + mon_host = 198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13 + auth_client_required = cephx + auth_cluster_required = cephx + auth_service_required = cephx + ms_bind_ipv4 = true + ms_bind_ipv6 = false + + [client.rgw.syd1] + rgw_realm = unkin + rgw_zonegroup = au + rgw_zone = syd1 + rgw_frontends = beast port=7480 + log_to_file = false + log_to_stderr = true + err_to_stderr = true diff --git a/apps/base/ceph-system/deployment.yaml b/apps/base/ceph-system/deployment.yaml new file mode 100644 index 0000000..d1fe170 --- /dev/null +++ b/apps/base/ceph-system/deployment.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rgw + namespace: ceph-system + labels: + app.kubernetes.io/name: rgw + app.kubernetes.io/component: rgw + app.kubernetes.io/version: v18.2.8 + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app.kubernetes.io/name: rgw + template: + metadata: + labels: + app.kubernetes.io/name: rgw + app.kubernetes.io/component: rgw + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 167 + runAsGroup: 167 + fsGroup: 167 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: rgw + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: rgw + containers: + - name: rgw + image: quay.io/ceph/ceph:v18.2.8 + imagePullPolicy: IfNotPresent + command: + - radosgw + - -f + - --cluster + - ceph + - --name + - client.rgw.syd1 + - --setuser + - ceph + - --setgroup + - ceph + ports: + - name: http + containerPort: 7480 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + readOnly: true + - name: keyring + mountPath: /var/lib/ceph/radosgw/ceph-rgw.syd1 + readOnly: true + - name: run + mountPath: /var/run/ceph + - name: log + mountPath: /var/log/ceph + - name: tmp + mountPath: /tmp + volumes: + - name: ceph-conf + configMap: + name: ceph-conf + - name: keyring + secret: + secretName: rgw-keyring + items: + - key: keyring + path: keyring + - name: run + emptyDir: {} + - name: log + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/apps/base/ceph-system/kustomization.yaml b/apps/base/ceph-system/kustomization.yaml new file mode 100644 index 0000000..18bd7ff --- /dev/null +++ b/apps/base/ceph-system/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - deployment.yaml + - service.yaml + - pdb.yaml diff --git a/apps/base/ceph-system/namespace.yaml b/apps/base/ceph-system/namespace.yaml new file mode 100644 index 0000000..31432d9 --- /dev/null +++ b/apps/base/ceph-system/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ceph-system diff --git a/apps/base/ceph-system/pdb.yaml b/apps/base/ceph-system/pdb.yaml new file mode 100644 index 0000000..c83eccf --- /dev/null +++ b/apps/base/ceph-system/pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rgw + namespace: ceph-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: rgw diff --git a/apps/base/ceph-system/service.yaml b/apps/base/ceph-system/service.yaml new file mode 100644 index 0000000..93d7174 --- /dev/null +++ b/apps/base/ceph-system/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: rgw + namespace: ceph-system + labels: + app.kubernetes.io/name: rgw +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: rgw + ports: + - name: http + port: 7480 + targetPort: http diff --git a/apps/base/ceph-system/vaultauth.yaml b/apps/base/ceph-system/vaultauth.yaml new file mode 100644 index 0000000..bea2fb9 --- /dev/null +++ b/apps/base/ceph-system/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: ceph-system +spec: + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default + allowedNamespaces: + - ceph-system + kubernetes: + role: default + serviceAccount: default + audiences: + - vault + tokenExpirationSeconds: 600 diff --git a/apps/base/ceph-system/vaultstaticsecret.yaml b/apps/base/ceph-system/vaultstaticsecret.yaml new file mode 100644 index 0000000..b10d6ed --- /dev/null +++ b/apps/base/ceph-system/vaultstaticsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: rgw-keyring + namespace: ceph-system +spec: + vaultAuthRef: default + mount: kv + type: kv-v2 + path: kubernetes/namespace/ceph-system/default/rgw-keyring + refreshAfter: 5m + hmacSecretData: true + destination: + name: rgw-keyring + create: true + overwrite: true diff --git a/apps/overlays/au-syd1/ceph-system/kustomization.yaml b/apps/overlays/au-syd1/ceph-system/kustomization.yaml new file mode 100644 index 0000000..eeaf1e9 --- /dev/null +++ b/apps/overlays/au-syd1/ceph-system/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/ceph-system diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index 2376860..515df7e 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -20,6 +20,7 @@ spec: - path: apps/overlays/*/cattle-system - path: apps/overlays/*/cert-manager - path: apps/overlays/*/certificates + - path: apps/overlays/*/ceph-system - path: apps/overlays/*/cephrgw-system - path: apps/overlays/*/cnpg-system - path: apps/overlays/*/consul