From 90f793464b5a2a8c3db0763aa65e667d0065daf5 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Thu, 19 Mar 2026 01:29:31 +1100
Subject: [PATCH] feat: migrate CSI drivers to dedicated storage project (#45)

- Migrate csi-cephfs from Terraform to ArgoCD
- Migrate csi-cephrbd from Terraform to ArgoCD
- Create dedicated storage project and ApplicationSet for CSI drivers
- Add csi-* pattern matching in storage ApplicationSet
- Remove CSI apps from platform project to separate concerns

Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/45
---
 apps/base/csi-cephfs/kustomization.yaml       |  9 ++
 apps/base/csi-cephfs/namespace.yaml           |  5 ++
 apps/base/csi-cephfs/storageclass.yaml        | 83 +++++++++++++++++++
 apps/base/csi-cephfs/vaultauth.yaml           | 18 ++++
 apps/base/csi-cephfs/vaultstaticsecret.yaml   | 15 ++++
 apps/base/csi-cephrbd/kustomization.yaml      |  9 ++
 apps/base/csi-cephrbd/namespace.yaml          |  5 ++
 apps/base/csi-cephrbd/storageclass.yaml       | 39 +++++++++
 apps/base/csi-cephrbd/vaultauth.yaml          | 18 ++++
 apps/base/csi-cephrbd/vaultstaticsecret.yaml  | 15 ++++
 .../au-syd1/csi-cephfs/kustomization.yaml     | 14 ++++
 apps/overlays/au-syd1/csi-cephfs/values.yaml  | 41 +++++++++
 .../au-syd1/csi-cephrbd/kustomization.yaml    | 14 ++++
 apps/overlays/au-syd1/csi-cephrbd/values.yaml | 48 +++++++++++
 argocd/applicationsets/storage.yaml           | 29 +++++++
 argocd/projects/storage.yaml                  | 32 +++++++
 16 files changed, 394 insertions(+)
 create mode 100644 apps/base/csi-cephfs/kustomization.yaml
 create mode 100644 apps/base/csi-cephfs/namespace.yaml
 create mode 100644 apps/base/csi-cephfs/storageclass.yaml
 create mode 100644 apps/base/csi-cephfs/vaultauth.yaml
 create mode 100644 apps/base/csi-cephfs/vaultstaticsecret.yaml
 create mode 100644 apps/base/csi-cephrbd/kustomization.yaml
 create mode 100644 apps/base/csi-cephrbd/namespace.yaml
 create mode 100644 apps/base/csi-cephrbd/storageclass.yaml
 create mode 100644 apps/base/csi-cephrbd/vaultauth.yaml
 create mode 100644 apps/base/csi-cephrbd/vaultstaticsecret.yaml
 create mode 100644 apps/overlays/au-syd1/csi-cephfs/kustomization.yaml
 create mode 100644 apps/overlays/au-syd1/csi-cephfs/values.yaml
 create mode 100644 apps/overlays/au-syd1/csi-cephrbd/kustomization.yaml
 create mode 100644 apps/overlays/au-syd1/csi-cephrbd/values.yaml
 create mode 100644 argocd/applicationsets/storage.yaml
 create mode 100644 argocd/projects/storage.yaml

diff --git a/apps/base/csi-cephfs/kustomization.yaml b/apps/base/csi-cephfs/kustomization.yaml
new file mode 100644
index 0000000..5849aad
--- /dev/null
+++ b/apps/base/csi-cephfs/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - storageclass.yaml
diff --git a/apps/base/csi-cephfs/namespace.yaml b/apps/base/csi-cephfs/namespace.yaml
new file mode 100644
index 0000000..11d6b94
--- /dev/null
+++ b/apps/base/csi-cephfs/namespace.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: csi-cephfs
diff --git a/apps/base/csi-cephfs/storageclass.yaml b/apps/base/csi-cephfs/storageclass.yaml
new file mode 100644
index 0000000..dafc1ca
--- /dev/null
+++ b/apps/base/csi-cephfs/storageclass.yaml
@@ -0,0 +1,83 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephfs-raid6-delete
+provisioner: cephfs.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+parameters:
+  clusterID: "cephfs_csi_ssd_ec_6_2"
+  fsName: "cephfs"
+  subVolumeGroup: csi_ssd_ec_6_2
+  csi.storage.k8s.io/provisioner-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-publish-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-publish-secret-namespace: "csi-cephfs"
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephfs-raid6-retain
+provisioner: cephfs.csi.ceph.com
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+parameters:
+  clusterID: "cephfs_csi_ssd_ec_6_2"
+  fsName: "cephfs"
+  subVolumeGroup: csi_ssd_ec_6_2
+  csi.storage.k8s.io/provisioner-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-publish-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-publish-secret-namespace: "csi-cephfs"
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephfs-raid5-delete
+provisioner: cephfs.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+parameters:
+  clusterID: "cephfs_csi_ssd_ec_4_1"
+  fsName: "cephfs"
+  subVolumeGroup: csi_ssd_ec_4_1
+  csi.storage.k8s.io/provisioner-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-publish-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-publish-secret-namespace: "csi-cephfs"
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephfs-raid5-retain
+provisioner: cephfs.csi.ceph.com
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+parameters:
+  clusterID: "cephfs_csi_ssd_ec_4_1"
+  fsName: "cephfs"
+  subVolumeGroup: csi_ssd_ec_4_1
+  csi.storage.k8s.io/provisioner-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephfs"
+  csi.storage.k8s.io/controller-publish-secret-name: "csi-cephfs-secret"
+  csi.storage.k8s.io/controller-publish-secret-namespace: "csi-cephfs"
diff --git a/apps/base/csi-cephfs/vaultauth.yaml b/apps/base/csi-cephfs/vaultauth.yaml
new file mode 100644
index 0000000..c4b2551
--- /dev/null
+++ b/apps/base/csi-cephfs/vaultauth.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: ceph-csi-cephfs
+  namespace: csi-cephfs
+spec:
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
+  allowedNamespaces:
+    - csi-cephfs
+  kubernetes:
+    role: ceph-csi
+    serviceAccount: ceph-csi-cephfs-csi-cephfs-provisioner
+    audiences:
+      - vault
+    tokenExpirationSeconds: 600
diff --git a/apps/base/csi-cephfs/vaultstaticsecret.yaml b/apps/base/csi-cephfs/vaultstaticsecret.yaml
new file mode 100644
index 0000000..a8101d4
--- /dev/null
+++ b/apps/base/csi-cephfs/vaultstaticsecret.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: csi-cephfs-secret
+  namespace: csi-cephfs
+spec:
+  vaultAuthRef: ceph-csi-cephfs
+  mount: kv
+  type: kv-v2
+  path: service/kubernetes/au/syd1/csi/ceph-cephfs-secret
+  refreshAfter: 5m
+  destination:
+    name: csi-cephfs-secret
+    create: true
diff --git a/apps/base/csi-cephrbd/kustomization.yaml b/apps/base/csi-cephrbd/kustomization.yaml
new file mode 100644
index 0000000..5849aad
--- /dev/null
+++ b/apps/base/csi-cephrbd/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - storageclass.yaml
diff --git a/apps/base/csi-cephrbd/namespace.yaml b/apps/base/csi-cephrbd/namespace.yaml
new file mode 100644
index 0000000..131483c
--- /dev/null
+++ b/apps/base/csi-cephrbd/namespace.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: csi-cephrbd
diff --git a/apps/base/csi-cephrbd/storageclass.yaml b/apps/base/csi-cephrbd/storageclass.yaml
new file mode 100644
index 0000000..3a7557f
--- /dev/null
+++ b/apps/base/csi-cephrbd/storageclass.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephrbd-fast-delete
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: rbd.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+parameters:
+  clusterID: "de96a98f-3d23-465a-a899-86d3d67edab8"
+  pool: "kubernetes"
+  imageFeatures: "layering"
+  csi.storage.k8s.io/provisioner-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephrbd"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephrbd"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephrbd"
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephrbd-fast-retain
+provisioner: rbd.csi.ceph.com
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+parameters:
+  clusterID: "de96a98f-3d23-465a-a899-86d3d67edab8"
+  pool: "kubernetes"
+  imageFeatures: "layering"
+  csi.storage.k8s.io/provisioner-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "csi-cephrbd"
+  csi.storage.k8s.io/controller-expand-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/controller-expand-secret-namespace: "csi-cephrbd"
+  csi.storage.k8s.io/node-stage-secret-name: "csi-rbd-secret"
+  csi.storage.k8s.io/node-stage-secret-namespace: "csi-cephrbd"
diff --git a/apps/base/csi-cephrbd/vaultauth.yaml b/apps/base/csi-cephrbd/vaultauth.yaml
new file mode 100644
index 0000000..e5c3a88
--- /dev/null
+++ b/apps/base/csi-cephrbd/vaultauth.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: ceph-csi-rbd
+  namespace: csi-cephrbd
+spec:
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
+  allowedNamespaces:
+    - csi-cephrbd
+  kubernetes:
+    role: ceph-csi
+    serviceAccount: ceph-csi-rbd-csi-rbd-provisioner
+    audiences:
+      - vault
+    tokenExpirationSeconds: 600
diff --git a/apps/base/csi-cephrbd/vaultstaticsecret.yaml b/apps/base/csi-cephrbd/vaultstaticsecret.yaml
new file mode 100644
index 0000000..02a06c8
--- /dev/null
+++ b/apps/base/csi-cephrbd/vaultstaticsecret.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: csi-rbd-secret
+  namespace: csi-cephrbd
+spec:
+  vaultAuthRef: ceph-csi-rbd
+  mount: kv
+  type: kv-v2
+  path: service/kubernetes/au/syd1/csi/ceph-rbd-secret
+  refreshAfter: 5m
+  destination:
+    name: csi-rbd-secret
+    create: true
diff --git a/apps/overlays/au-syd1/csi-cephfs/kustomization.yaml b/apps/overlays/au-syd1/csi-cephfs/kustomization.yaml
new file mode 100644
index 0000000..398f3e0
--- /dev/null
+++ b/apps/overlays/au-syd1/csi-cephfs/kustomization.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/csi-cephfs
+
+helmCharts:
+  - name: ceph-csi-cephfs
+    repo: https://ceph.github.io/csi-charts
+    version: "3.15.0"
+    releaseName: ceph-csi-cephfs
+    namespace: csi-cephfs
+    valuesFile: values.yaml
diff --git a/apps/overlays/au-syd1/csi-cephfs/values.yaml b/apps/overlays/au-syd1/csi-cephfs/values.yaml
new file mode 100644
index 0000000..9c316a5
--- /dev/null
+++ b/apps/overlays/au-syd1/csi-cephfs/values.yaml
@@ -0,0 +1,41 @@
+csiConfig:
+  # raid6, 75% space efficiency
+  - clusterID: "cephfs_csi_ssd_ec_6_2"
+    monitors:
+      - "198.18.23.9:6789"
+      - "198.18.23.10:6789"
+      - "198.18.23.11:6789"
+      - "198.18.23.12:6789"
+      - "198.18.23.13:6789"
+    cephFS:
+      subvolumeGroup: csi_ssd_ec_6_2
+
+  # raid5, 80% space efficiency
+  - clusterID: "cephfs_csi_ssd_ec_4_1"
+    monitors:
+      - "198.18.23.9:6789"
+      - "198.18.23.10:6789"
+      - "198.18.23.11:6789"
+      - "198.18.23.12:6789"
+      - "198.18.23.13:6789"
+    cephFS:
+      subvolumeGroup: csi_ssd_ec_4_1
+
+# Secret containing Ceph credentials
+secret:
+  create: false
+  name: "csi-cephfs-secret"
+
+# Storage class configuration (disabled - managed separately when needed)
+storageClass:
+  create: false
+
+# Provisioner configuration
+provisioner:
+  name: csi-cephfs-provisioner
+  replicaCount: 3
+
+# Image versions
+image:
+  repository: quay.io/cephcsi/cephcsi
+  tag: v3.15.0
diff --git a/apps/overlays/au-syd1/csi-cephrbd/kustomization.yaml b/apps/overlays/au-syd1/csi-cephrbd/kustomization.yaml
new file mode 100644
index 0000000..cfa2538
--- /dev/null
+++ b/apps/overlays/au-syd1/csi-cephrbd/kustomization.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/csi-cephrbd
+
+helmCharts:
+  - name: ceph-csi-rbd
+    repo: https://ceph.github.io/csi-charts
+    version: "3.15.1"
+    releaseName: ceph-csi-rbd
+    namespace: csi-cephrbd
+    valuesFile: values.yaml
diff --git a/apps/overlays/au-syd1/csi-cephrbd/values.yaml b/apps/overlays/au-syd1/csi-cephrbd/values.yaml
new file mode 100644
index 0000000..f2f230a
--- /dev/null
+++ b/apps/overlays/au-syd1/csi-cephrbd/values.yaml
@@ -0,0 +1,48 @@
+csiConfig:
+  - clusterID: "de96a98f-3d23-465a-a899-86d3d67edab8"
+    monitors:
+      - "198.18.23.9:6789"
+      - "198.18.23.10:6789"
+      - "198.18.23.11:6789"
+      - "198.18.23.12:6789"
+      - "198.18.23.13:6789"
+
+# Secret containing Ceph credentials
+secret:
+  create: false
+  name: "csi-rbd-secret"
+
+# Storage class configuration (disabled - managed separately)
+storageClass:
+  create: false
+
+# CSI driver configuration
+csiDriver:
+  name: "rbd.csi.ceph.com"
+
+# Node plugin configuration
+nodeplugin:
+  name: csi-rbdplugin
+  updateStrategy: RollingUpdate
+
+# Provisioner configuration
+provisioner:
+  name: csi-rbd-provisioner
+  replicaCount: 3
+
+# Image versions
+image:
+  repository: quay.io/cephcsi/cephcsi
+  tag: v3.15.0
+
+# Enable snapshots
+snapshotter:
+  image:
+    repository: registry.k8s.io/sig-storage/csi-snapshotter
+    tag: v8.0.1
+
+# Enable volume expansion
+resizer:
+  image:
+    repository: registry.k8s.io/sig-storage/csi-resizer
+    tag: v1.12.0
diff --git a/argocd/applicationsets/storage.yaml b/argocd/applicationsets/storage.yaml
new file mode 100644
index 0000000..6994771
--- /dev/null
+++ b/argocd/applicationsets/storage.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: storage-apps
+  namespace: argocd
+spec:
+  generators:
+  - git:
+      repoURL: https://git.unkin.net/unkin/argocd-apps
+      revision: HEAD
+      directories:
+      - path: apps/overlays/*/csi-*
+  template:
+    metadata:
+      name: 'storage-{{path[3]}}'  # cluster-app format (e.g., storage-csi-cephfs)
+    spec:
+      project: storage
+      source:
+        repoURL: https://git.unkin.net/unkin/argocd-apps
+        targetRevision: HEAD
+        path: '{{path}}'
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: '{{path[3]}}'  # Use directory name as namespace
+      syncPolicy:
+        automated:
+          prune: true
+          selfHeal: true
diff --git a/argocd/projects/storage.yaml b/argocd/projects/storage.yaml
new file mode 100644
index 0000000..f8013ec
--- /dev/null
+++ b/argocd/projects/storage.yaml
@@ -0,0 +1,32 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: storage
+  namespace: argocd
+spec:
+  description: Storage infrastructure and CSI drivers
+  sourceRepos:
+    - https://git.unkin.net/unkin/argocd-apps
+    - https://ceph.github.io/csi-charts
+  destinations:
+    - namespace: 'csi-*'
+      server: https://kubernetes.default.svc
+  clusterResourceWhitelist:
+    - group: ''
+      kind: Namespace
+    - group: 'rbac.authorization.k8s.io'
+      kind: ClusterRole
+    - group: 'rbac.authorization.k8s.io'
+      kind: ClusterRoleBinding
+    - group: 'apiextensions.k8s.io'
+      kind: CustomResourceDefinition
+    - group: 'storage.k8s.io'
+      kind: StorageClass
+    - group: 'storage.k8s.io'
+      kind: CSIDriver
+    - group: 'scheduling.k8s.io'
+      kind: PriorityClass
+  namespaceResourceWhitelist:
+    - group: '*'
+      kind: '*'