Merge pull request #422 from mojaloop/feat/rook-ceph-cluster

Feat/rook ceph cluster

gitops/applications/base/gitlab-pre/kustomization.yaml

@@ -12,4 +12,4 @@ helmCharts:
   version: ${ARGOCD_ENV_redis_helm_version}
   repo: oci://registry-1.docker.io/bitnamicharts/
   valuesFile: redis-values.yaml
-  namespace: ${ARGOCD_ENV_gitlab_namespace}    
+  namespace: ${ARGOCD_ENV_gitlab_namespace}

gitops/applications/base/rook-ceph/ceph-blockpool-crs.yaml

@@ -0,0 +1,11 @@
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: ceph-blockpool
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  failureDomain: host
+  replicated:
+    size: ${ARGOCD_ENV_objects_replica_count}
+    requireSafeReplicaSize: true
+    #targetSizeRatio: .5

gitops/applications/base/rook-ceph/ceph-filsystem-crs.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystemSubVolumeGroup
+metadata:
+  name: ceph-filesystem-csi
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  name: csi
+  filesystemName: ceph-filesystem
+  pinning:
+    distributed: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+  name: ceph-filesystem
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  metadataPool:
+    replicated:
+      size: 3
+      requireSafeReplicaSize: true
+  dataPools:
+    - name: replicated
+      failureDomain: host
+      replicated:
+        size: ${ARGOCD_ENV_objects_replica_count}
+        requireSafeReplicaSize: true
+  preserveFilesystemOnDelete: true
+  metadataServer:
+    activeCount: 1
+    activeStandby: true
+    priorityClassName: system-cluster-critical

gitops/applications/base/rook-ceph/ceph-objectstore-crs.yaml

@@ -0,0 +1,21 @@
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: ceph-objectstore
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: ${ARGOCD_ENV_objects_replica_count}
+  dataPool:
+    failureDomain: host
+    erasureCoded:
+      dataChunks: 2
+      codingChunks: 1
+  gateway:
+    port: 80
+    # securePort: 443
+    # sslCertificateRef:
+    instances: 1
+    priorityClassName: system-cluster-critical

gitops/applications/base/rook-ceph/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ceph-blockpool-crs.yaml
+  - ceph-objectstore-crs.yaml
+  - ceph-filsystem-crs.yaml
+  - storageclass.yaml

gitops/applications/base/rook-ceph/storageclass.yaml

@@ -0,0 +1,60 @@
+# Object storage
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-bucket
+provisioner: rook-ceph.ceph.rook.io/bucket
+reclaimPolicy: Delete
+volumeBindingMode: "Immediate"
+parameters:
+  objectStoreName: ceph-objectstore
+  objectStoreNamespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  region: "us-east-1"
+
+---
+# Ceph Filesystem
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-filesystem
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: rook-ceph.cephfs.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+volumeBindingMode: "Immediate"
+parameters:
+  clusterID: "${ARGOCD_ENV_rook_ceph_namespace}"
+  fsName: ceph-filesystem
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/fstype: ext4
+
+---
+# Ceph Block
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-block
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: rook-ceph.rbd.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+volumeBindingMode: "Immediate"
+parameters:
+  clusterID: "${ARGOCD_ENV_rook_ceph_namespace}"
+  pool: ceph-blockpool
+  imageFormat: "2"
+  imageFeatures: layering
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+  csi.storage.k8s.io/node-stage-secret-namespace: "${ARGOCD_ENV_rook_ceph_namespace}"
+  csi.storage.k8s.io/fstype: ext4

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/host/ceph-cluster-crs.yaml

@@ -0,0 +1,23 @@
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  cephVersion:
+    image: quay.io/ceph/ceph:${ARGOCD_ENV_ceph_image_version}
+    allowUnsupported: false
+  dataDirHostPath: /var/lib/rook
+  mgr:
+    count: 2
+    allowMultiplePerNode: false
+  mon:
+    count: 3
+    allowMultiplePerNode: false
+  monitoring:
+    enabled: true
+  dashboard:
+    enabled: true
+  storage:
+    useAllNodes: true
+    useAllDevices: true

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/host/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ceph-cluster-crs.yaml

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/pvc/aws-ebs-csi-driver-values.yaml

@@ -0,0 +1,39 @@
+# Custom values for aws-ebs-csi-driver.
+# Use old CSIDriver without an fsGroupPolicy set
+# Intended for use with older clusters that cannot easily replace the CSIDriver object
+# This parameter should always be false for new installations
+useOldCSIDriver: false
+# Deploy EBS CSI Driver without controller and associated resources
+nodeComponentOnly: false
+
+sidecars:
+  attacher:
+    securityContext:
+      allowPrivilegeEscalation: true
+  snapshotter:
+    forceEnable: false
+    securityContext:
+      allowPrivilegeEscalation: false
+  resizer:
+    securityContext:
+      allowPrivilegeEscalation: false
+  volumemodifier:
+    securityContext:
+      allowPrivilegeEscalation: false
+
+node:
+  kubeletPath: "${ARGOCD_ENV_kubelet_dir_path}"
+  securityContext:
+    runAsNonRoot: false
+controller:
+  # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass.
+  # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set
+  replicaCount: "${ARGOCD_ENV_csi_driver_replicas}"
+  defaultFsType: ext4
+  batching: true
+  volumeModificationFeature:
+    enabled: false
+  env: []
+  extraVolumeTags:
+    Name: "rook-volume-${ARGOCD_ENV_cluster_domain}"
+  enableMetrics: false

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/pvc/ceph-cluster-crs.yaml

@@ -0,0 +1,109 @@
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  cephVersion:
+    image: quay.io/ceph/ceph:${ARGOCD_ENV_ceph_image_version}
+    allowUnsupported: false
+  dataDirHostPath: /var/lib/rook
+  monitoring:
+    enabled: true
+  dashboard:
+    enabled: true
+  mgr:
+    count: 2
+    allowMultiplePerNode: false
+  mon:
+    count: 3
+    allowMultiplePerNode: false
+    volumeClaimTemplate:
+      spec:
+        storageClassName: "${ARGOCD_ENV_ceph_osd_volumes_storage_class}-csi"
+        resources:
+          requests:
+            storage: "${ARGOCD_ENV_ceph_mon_volume_size}"
+  storage:
+    allowDeviceClassUpdate: false # whether to allow changing the device class of an OSD after it is created
+    allowOsdCrushWeightUpdate: true # whether to allow resizing the OSD crush weight after osd pvc is increased
+    storageClassDeviceSets:
+      - name: set1
+        # The number of OSDs to create from this device set
+        count: ${ARGOCD_ENV_ceph_osd_count}
+        portable: true
+        tuneDeviceClass: true
+        tuneFastDeviceClass: false
+        encrypted: false
+        placement:
+          topologySpreadConstraints:
+            - maxSkew: 1
+              topologyKey: kubernetes.io/hostname
+              whenUnsatisfiable: ScheduleAnyway
+              labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - rook-ceph-osd
+        preparePlacement:
+          podAntiAffinity:
+            preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                      - key: app
+                        operator: In
+                        values:
+                          - rook-ceph-osd
+                      - key: app
+                        operator: In
+                        values:
+                          - rook-ceph-osd-prepare
+                  topologyKey: kubernetes.io/hostname
+          topologySpreadConstraints:
+            - maxSkew: 1
+              # IMPORTANT: If you don't have zone labels, change this to another key such as kubernetes.io/hostname
+              topologyKey: topology.kubernetes.io/zone
+              whenUnsatisfiable: DoNotSchedule
+              labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - rook-ceph-osd-prepare
+        resources:
+        # These are the OSD daemon limits. For OSD prepare limits, see the separate section below for "prepareosd" resources
+        #   limits:
+        #     memory: "4Gi"
+        #   requests:
+        #     cpu: "500m"
+        #     memory: "4Gi"
+        volumeClaimTemplates:
+          - metadata:
+              name: data
+            spec:
+              resources:
+                requests:
+                  storage: "${ARGOCD_ENV_ceph_volume_size_per_osd}"
+              storageClassName: "${ARGOCD_ENV_ceph_osd_volumes_storage_class}-csi"
+              volumeMode: Block
+              accessModes:
+                - ReadWriteOnce
+    onlyApplyOSDPlacement: false
+  resources:
+  #  prepareosd:
+  #    requests:
+  #      cpu: "200m"
+  #      memory: "200Mi"
+  priorityClassNames:
+    # If there are multiple nodes available in a failure domain (e.g. zones), the
+    # mons and osds can be portable and set the system-cluster-critical priority class.
+    mon: system-node-critical
+    osd: system-node-critical
+    mgr: system-cluster-critical
+  disruptionManagement:
+    managePodBudgets: true
+    osdMaintenanceTimeout: 30
+    pgHealthCheckTimeout: 0

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/pvc/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - volumes-storageclass.yaml
+  - ceph-cluster-crs.yaml
+
+helmCharts:
+  - name: aws-ebs-csi-driver
+    releaseName: aws-ebs-csi-driver
+    repo: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+    namespace: kube-system
+    valuesFile: aws-ebs-csi-driver-values.yaml

gitops/applications/overlays/cloud_provider/aws/rook-ceph-cluster/pvc/volumes-storageclass.yaml

@@ -0,0 +1,13 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: "${ARGOCD_ENV_ceph_osd_volumes_storage_class}-csi"
+provisioner: ebs.csi.aws.com
+parameters:
+  type: ${ARGOCD_ENV_ceph_osd_volumes_storage_class}
+  encrypted: "true"
+  #iops: "3000"
+  #throughput: "125"
+  #csi.storage.k8s.io/fstype: xfs
+reclaimPolicy: ${ARGOCD_ENV_cloud_pv_reclaim_policy}
+volumeBindingMode: WaitForFirstConsumer