CTO-441 Enable ceph bucket monitoring (#362)

docs/monitoring/database-metrics.md

@@ -0,0 +1,11 @@
+# Context
+The database (eg. mysql/mongo) may run inside the mojaloop cluster or may run as managed database (e.g. AWS RDS). In both cases, we want to gather database metrics for operational visibility
+
+# Problem
+How do we show the same metrics to ops team when the database is running as managed instance (e.g. AWS RDS). 
+
+# Solution
+
+In case of self managed database, the exporter runs as a side car container with the database container in the same k8s pod. When the database runs as external managed service (eg. AWS RDS), we deploy a standalone exporter instance. This exporter instance pulls the metrics data from the database and converts them to prometheus format. 
+
+![diagram](./database-metrics-architecture.svg) 

gitops/applications/base/monitoring-post-config/dashboards-rook-ceph.yaml

@@ -54,4 +54,19 @@ spec:
   grafanaCom:
     id: 5342
     revision: 9
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: ceph-objectstore
+spec:
+  folder: storage
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: "${ARGOCD_ENV_dashboard_datasource_name}"
+  url: "https://raw.githubusercontent.com/mojaloop/iac-modules/${ARGOCD_ENV_monitoring_application_gitrepo_tag}/assets/grafana-dashboards/ceph-objectstore.json"
+
 ---

gitops/applications/base/utils-post-config/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
   - rbac.yaml
   - crossplane-providers.yaml
+  - rook-ceph-objectstore-exporter.yaml
 
 secretGenerator:
   - name: terraformrc

gitops/applications/base/utils-post-config/rook-ceph-objectstore-exporter.yaml

@@ -0,0 +1,113 @@
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: ceph-objectstore-exporter-user
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+spec:
+  store: ceph-objectstore
+  displayName: ceph-objectstore-exporter-user
+  clusterNamespace: ${ARGOCD_ENV_rook_ceph_namespace} 
+  capabilities:
+    bucket: read
+    metadata: read
+    usage: read
+    user: read
+
+
+--- 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ceph-objectstore-exporter
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+  labels:
+    app.kubernetes.io/name: ceph-objectstore-exporter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ceph-objectstore-exporter
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ceph-objectstore-exporter
+    spec:
+      containers:
+      - image: ghcr.io/pando85/radosgw_usage_exporter:latest
+        env:
+        - name: ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: AccessKey
+              name: rook-ceph-object-user-ceph-objectstore-ceph-objectstore-exporter-user
+        - name: SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              key: SecretKey
+              name: rook-ceph-object-user-ceph-objectstore-ceph-objectstore-exporter-user
+        - name: RADOSGW_SERVER
+          valueFrom:
+            secretKeyRef:
+              key: Endpoint
+              name: rook-ceph-object-user-ceph-objectstore-ceph-objectstore-exporter-user
+        args:
+        - --insecure
+        name: exporter
+        ports:
+        - containerPort: 9242
+          name: http
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 400m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
+        livenessProbe:
+          tcpSocket:
+            port: http
+        readinessProbe:
+          tcpSocket:
+            port: http
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ceph-objectstore-exporter
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+  labels:
+    app.kubernetes.io/name: ceph-objectstore-exporter
+spec:
+  selector:
+    app.kubernetes.io/name: ceph-objectstore-exporter
+  ports:
+  - name: http
+    port: 9242
+    protocol: TCP
+    targetPort: http
+--- 
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ceph-objectstore-exporter
+  namespace: ${ARGOCD_ENV_rook_ceph_namespace}
+  labels:
+    app.kubernetes.io/name: ceph-objectstore-exporter
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ceph-objectstore-exporter
+  endpoints:
+  - interval: 1m
+    port: http
+

gitops/argo-apps/base/monitoring-post-config.yaml

@@ -31,8 +31,7 @@ spec:
 
   source:
     repoURL: ${ARGOCD_ENV_argocd_repo_url}
-    # targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag}
-    targetRevision: feature/ccv2-common-branch
+    targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag}
     path: gitops/applications/base/monitoring-post-config
     plugin:
       name: envsubst

gitops/argo-apps/base/monitoring-pre.yaml

@@ -31,8 +31,7 @@ spec:
 
   source:
     repoURL: ${ARGOCD_ENV_argocd_repo_url}
-    # targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag} 
-    targetRevision: feature/ccv2-common-branch
+    targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag} 
     path: gitops/applications/base/monitoring-pre
     plugin:
       name: envsubst

gitops/argo-apps/base/monitoring.yaml

@@ -57,8 +57,7 @@ spec:
 
   source:
     repoURL: ${ARGOCD_ENV_argocd_repo_url}
-    # targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag}
-    targetRevision: feature/ccv2-common-branch
+    targetRevision: ${ARGOCD_ENV_monitoring_application_gitrepo_tag}
     path: gitops/applications/base/monitoring
     plugin:
       name: envsubst

gitops/argo-apps/base/utils-post-config.yaml

@@ -47,3 +47,6 @@ spec:
 
         - name: "crossplane_namespace"
           value: "${ARGOCD_ENV_utils_crossplane_namespace}"
+
+        - name: "rook_ceph_namespace"
+          value: "${ARGOCD_ENV_utils_rook_ceph_namespace}"

terraform/config-params/ccnew-config/env-onboard-config/netbird.tf

@@ -44,6 +44,11 @@ resource "netbird_group" "env_backtunnel" {
     create_before_destroy = true
   }
 }
+
+resource "netbird_group" "env_backtunnel" {
+  count = var.k8s_cluster_type == "eks" ? 1 : 0
+  name  = "${var.env_name}-backtunnel"
+}
 #route to allow private traffic into en k8s network from cc user group and the env_users group, env gw is the gateway peer
 resource "netbird_route" "env_k8s" {
   description = "${var.env_name}-k8s"