ceph-system: run radosgw as a deployment (#583)
Part of #582. RGW runs on VMs (ausyd1nxvm2115-2119) and needs to move into k8s under GitOps; this adds the `ceph-system` namespace with an in-cluster RGW behind a ClusterIP service. Cutover comes later. - add `ceph-system` app to the platform applicationset - add ceph.conf configmap with a shared `client.rgw.syd1` section and explicit keyring path - add rgw keyring from vault `kubernetes/namespace/ceph-system/default/rgw-keyring` (`keyring`) - add 3-replica `quay.io/ceph/ceph:v18.2.8` radosgw deployment, non-root, read-only root, startup probe, hard per-node spread, PDB + VPA - add ClusterIP service `rgw` on 7480 Requires unkin/terraform-artifactapi#78 merged first. Reviewed-on: #583 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #583.
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ceph-conf
|
||||
namespace: ceph-system
|
||||
data:
|
||||
ceph.conf: |
|
||||
[global]
|
||||
fsid = de96a98f-3d23-465a-a899-86d3d67edab8
|
||||
mon_host = 198.18.23.9,198.18.23.10,198.18.23.11,198.18.23.12,198.18.23.13
|
||||
auth_client_required = cephx
|
||||
auth_cluster_required = cephx
|
||||
auth_service_required = cephx
|
||||
ms_bind_ipv4 = true
|
||||
ms_bind_ipv6 = false
|
||||
|
||||
[client.rgw.syd1]
|
||||
rgw_realm = unkin
|
||||
rgw_zonegroup = au
|
||||
rgw_zone = syd1
|
||||
rgw_frontends = beast port=7480
|
||||
keyring = /var/lib/ceph/radosgw/ceph-rgw.syd1/keyring
|
||||
log_to_file = false
|
||||
log_to_stderr = true
|
||||
err_to_stderr = true
|
||||
@@ -0,0 +1,125 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rgw
|
||||
namespace: ceph-system
|
||||
labels:
|
||||
app.kubernetes.io/name: rgw
|
||||
app.kubernetes.io/component: rgw
|
||||
app.kubernetes.io/version: v18.2.8
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
spec:
|
||||
replicas: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
maxSurge: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rgw
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: rgw
|
||||
app.kubernetes.io/component: rgw
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 167
|
||||
runAsGroup: 167
|
||||
fsGroup: 167
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rgw
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rgw
|
||||
containers:
|
||||
- name: rgw
|
||||
image: quay.io/ceph/ceph:v18.2.8
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- radosgw
|
||||
- -f
|
||||
- --cluster
|
||||
- ceph
|
||||
- --name
|
||||
- client.rgw.syd1
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 7480
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
periodSeconds: 20
|
||||
failureThreshold: 6
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
readOnly: true
|
||||
- name: keyring
|
||||
mountPath: /var/lib/ceph/radosgw/ceph-rgw.syd1
|
||||
readOnly: true
|
||||
- name: run
|
||||
mountPath: /var/run/ceph
|
||||
- name: log
|
||||
mountPath: /var/log/ceph
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: ceph-conf
|
||||
configMap:
|
||||
name: ceph-conf
|
||||
- name: keyring
|
||||
secret:
|
||||
secretName: rgw-keyring
|
||||
items:
|
||||
- key: keyring
|
||||
path: keyring
|
||||
- name: run
|
||||
emptyDir: {}
|
||||
- name: log
|
||||
emptyDir: {}
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- configmap.yaml
|
||||
- vaultauth.yaml
|
||||
- vaultstaticsecret.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- pdb.yaml
|
||||
- vpa.yaml
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: ceph-system
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: rgw
|
||||
namespace: ceph-system
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rgw
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: rgw
|
||||
namespace: ceph-system
|
||||
labels:
|
||||
app.kubernetes.io/name: rgw
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: rgw
|
||||
ports:
|
||||
- name: http
|
||||
port: 7480
|
||||
targetPort: http
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: default
|
||||
namespace: ceph-system
|
||||
spec:
|
||||
method: kubernetes
|
||||
mount: k8s/au/syd1
|
||||
vaultConnectionRef: vso-system/default
|
||||
allowedNamespaces:
|
||||
- ceph-system
|
||||
kubernetes:
|
||||
role: default
|
||||
serviceAccount: default
|
||||
audiences:
|
||||
- vault
|
||||
tokenExpirationSeconds: 600
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
# Seed kv/kubernetes/namespace/ceph-system/default/rgw-keyring field `keyring` with the client.rgw.syd1 keyring.
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: rgw-keyring
|
||||
namespace: ceph-system
|
||||
spec:
|
||||
vaultAuthRef: default
|
||||
mount: kv
|
||||
type: kv-v2
|
||||
path: kubernetes/namespace/ceph-system/default/rgw-keyring
|
||||
refreshAfter: 5m
|
||||
hmacSecretData: true
|
||||
destination:
|
||||
name: rgw-keyring
|
||||
create: true
|
||||
overwrite: true
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
apiVersion: autoscaling.k8s.io/v1
|
||||
kind: VerticalPodAutoscaler
|
||||
metadata:
|
||||
name: rgw-vpa
|
||||
namespace: ceph-system
|
||||
spec:
|
||||
targetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: rgw
|
||||
updatePolicy:
|
||||
updateMode: "Off"
|
||||
Reference in New Issue
Block a user