Merge pull request #438 from mojaloop/feature/cto-696

Feature/cto 696
mojaloop · Dec 16, 2024 · 2eee7b9 · 2eee7b9
2 parents 10c2d8a + c6cbbd3
commit 2eee7b9
Show file tree

Hide file tree

Showing 11 changed files with 84 additions and 43 deletions.
diff --git a/.../overlays/cloud_provider/aws/xplane-provider-config/xr-definition-rds-aurora-replica.yaml b/.../overlays/cloud_provider/aws/xplane-provider-config/xr-definition-rds-aurora-replica.yaml
@@ -22,6 +22,9 @@ spec:
             spec:
               type: object
               properties:
+                id:
+                  type: string
+                  description: Database ID              
                 count:
                   type: integer
                   description: "The number of Aurora replicas to create."

diff --git a/gitops/applications/overlays/rdbms_provider/rds/helmcharts/dbmr/templates/mr.yaml b/gitops/applications/overlays/rdbms_provider/rds/helmcharts/dbmr/templates/mr.yaml
@@ -62,6 +62,7 @@ metadata:
     organization: Infitx
     author: devops   
 spec:
+  id: {{ printf "%s-%s" .Values.namePrefix  .Values.name }}
   compositionSelector:
     matchLabels:
       provider: aws

diff --git a/terraform/aws/support-svcs/deploy-rds-cluster/main.tf b/terraform/aws/support-svcs/deploy-rds-cluster/main.tf
@@ -1,3 +1,7 @@
+locals {
+  is_aurora = try(regex("aurora", var.engine),"") == "aurora" ? true : false
+}
+
 module "db_subnet_group" {
   source = "terraform-aws-modules/rds/aws//modules/db_subnet_group"
 
@@ -47,9 +51,10 @@ resource "aws_rds_cluster" "rds_cluster" {
 
   engine                    = var.engine
   engine_version            = var.engine_version
-  db_cluster_instance_class = var.instance_class
-  allocated_storage         = var.allocated_storage
-  storage_type              = var.storage_type
+  db_cluster_instance_class = local.is_aurora ?  null : var.instance_class
+  allocated_storage         = local.is_aurora ?  null : var.allocated_storage
+  storage_type              = local.is_aurora ?  null : var.storage_type
+  iops                      = local.is_aurora ?  null : var.iops
   storage_encrypted         = var.storage_encrypted
   kms_key_id                = var.kms_key_id
 
@@ -66,7 +71,7 @@ resource "aws_rds_cluster" "rds_cluster" {
 
   network_type = var.network_type
 
-  iops = var.iops
+
 
   allow_major_version_upgrade  = var.allow_major_version_upgrade
   apply_immediately            = var.apply_immediately
@@ -88,4 +93,13 @@ resource "aws_rds_cluster" "rds_cluster" {
 
   tags = var.tags
 
+}
+
+resource "aws_rds_cluster_instance" "cluster_instances" {
+  count              = local.is_aurora ? var.replicas : 0
+  identifier         = "${var.identifier}-${count.index}"
+  cluster_identifier = aws_rds_cluster.rds_cluster[0].id
+  instance_class     = var.instance_class
+  engine             = aws_rds_cluster.rds_cluster[0].engine
+  engine_version     = aws_rds_cluster.rds_cluster[0].engine_version
 }
diff --git a/terraform/aws/support-svcs/deploy-rds-cluster/variables.tf b/terraform/aws/support-svcs/deploy-rds-cluster/variables.tf
@@ -660,4 +660,9 @@ variable "master_user_password_rotation_schedule_expression" {
   description = "A cron() or rate() expression that defines the schedule for rotating your secret. Either automatically_after_days or schedule_expression must be specified."
   type        = string
   default     = null
+}
+
+variable "replicas"{
+  type        = number
+  default     = 2
 }
diff --git a/terraform/aws/support-svcs/deploy-rds/infra.tf b/terraform/aws/support-svcs/deploy-rds/infra.tf
@@ -21,6 +21,7 @@ module "rds" {
   iops                 = each.value.external_resource_config.storage_iops
   multi_az            = each.value.external_resource_config.multi_az
   skip_final_snapshot = each.value.external_resource_config.skip_final_snapshot
+  replicas            = each.value.external_resource_config.replicas
 
   db_name  = each.value.external_resource_config.db_name
   username = each.value.external_resource_config.username

diff --git a/terraform/ccnew/default-config/common-vars.yaml b/terraform/ccnew/default-config/common-vars.yaml
@@ -169,7 +169,7 @@ gitlab_postgres_rdbms_provider: "percona"
 #gitlab rds
 gitlab_rds_engine: "aurora-postgresql"
 gitlab_rds_engine_version: "'16'"
-gitlab_rds_replica_count: "'2'"
+gitlab_rds_replica_count: "'1'"
 gitlab_rds_instance_class: "db.t3.medium"
 gitlab_rds_storage_encrypted: "'true'"
 gitlab_rds_skip_final_snapshot: "'true'"
@@ -180,7 +180,7 @@ gitlab_db_storage_iops: "'5000'"
 #praefect rds
 praefect_rds_engine: "aurora-postgresql"
 praefect_rds_engine_version: "'16'"
-praefect_rds_replica_count: "'2'"
+praefect_rds_replica_count: "'1'"
 praefect_rds_instance_class: "db.t3.medium"
 praefect_rds_storage_encrypted: "'true'"
 praefect_rds_skip_final_snapshot: "'true'"
@@ -202,7 +202,7 @@ zitadel_db_storage_size: "5Gi"
 #Zitadel RDS specific
 zitadel_rds_engine: "aurora-postgresql"
 zitadel_rds_engine_version: "'16'"
-zitadel_rds_replica_count: "'2'"
+zitadel_rds_replica_count: "'1'"
 zitadel_rds_instance_class: "db.t3.medium"
 zitadel_rds_storage_encrypted: "'true'"
 zitadel_rds_skip_final_snapshot: "'true'"

diff --git a/terraform/gitops/stateful-resources/stateful-resources-config.tf b/terraform/gitops/stateful-resources/stateful-resources-config.tf
@@ -132,6 +132,7 @@ resource "local_file" "percona-crs" {
       mongod_replica_count              = each.value.logical_service_config.replica_count
       percona_server_mongodb_version    = each.value.resource_type == "mongodb" ? each.value.local_operator_config.percona_server_mongodb_version : ""
       percona_backup_mongodb_version    = each.value.resource_type == "mongodb" ? each.value.local_operator_config.percona_backup_mongodb_version : ""
+      additional_privileges             = each.value.resource_type == "mongodb" ? each.value.local_operator_config.additional_privileges : []
 
 
       ceph_percona_backup_bucket = var.ceph_percona_backup_bucket

diff --git a/...itops/stateful-resources/templates/stateful-resources/percona/mongodb/db-cluster.yaml.tpl b/...itops/stateful-resources/templates/stateful-resources/percona/mongodb/db-cluster.yaml.tpl
@@ -666,6 +666,12 @@ spec:
             - >
                echo "use ${database_name}" >> ~/init.js;       
                echo "db.createUser({user: \"${database_user}\",pwd: process.env.MONGODB_USER_PASSWORD,roles: [{ db: \"${database_name}\", role: \"readWrite\" }],mechanisms: [\"SCRAM-SHA-1\"]})" >> ~/init.js;
+%{ for privilege in additional_privileges ~}
+               echo "db.createRole({ role: \"additionalRole\", privileges: [{ resource: { db: \"${database_name}\", collection: \"${privilege.collection}\" }, actions: [\"${privilege.action}\"] }], roles: [] })" >> ~/init.js;
+%{ endfor ~}
+%{ if additional_privileges != [] ~}
+               echo "db.updateUser(\"${database_user}\", { roles: [ { db: \"${database_name}\", role: \"readWrite\" },{ role: \"additionalRole\", db: \"${database_user}\" }]})" >> ~/init.js;
+%{ endif ~}                 
                chmod +x ~/init.js;
                echo "running init.js";
                mongosh "mongodb://$${MONGODB_USER_ADMIN_USER}:$${MONGODB_USER_ADMIN_PASSWORD}@${cluster_name}-mongos" < ~/init.js

diff --git a/terraform/k8s/default-config/mojaloop-stateful-resources-local-operator.yaml b/terraform/k8s/default-config/mojaloop-stateful-resources-local-operator.yaml
@@ -348,6 +348,7 @@ bulk-mongodb:
       storage_class_name: longhorn
       service_port: 27017
       affinity_definition: {}
+    additional_privileges: []
 cep-mongodb:
   local_operator_config:
     cr_version: 1.16.2
@@ -389,6 +390,7 @@ cep-mongodb:
       storage_class_name: longhorn
       service_port: 27017
       affinity_definition: {}
+    additional_privileges: []      
 reporting-events-mongodb:
   local_operator_config:
     cr_version: 1.16.2
@@ -430,6 +432,9 @@ reporting-events-mongodb:
       storage_class_name: longhorn
       service_port: 27017
       affinity_definition: {}
+    additional_privileges:
+      - collection: reporting
+        action: "collMod"
 mojaloop-kafka:
   local_operator_config:
     resource_namespace: mojaloop
@@ -510,6 +515,7 @@ ttk-mongodb:
       storage_class_name: longhorn
       service_port: 27017
       affinity_definition: {}
+    additional_privileges: []      
 
 keycloak-db:
   local_operator_config:

diff --git a/terraform/k8s/default-config/mojaloop-stateful-resources-managed.yaml b/terraform/k8s/default-config/mojaloop-stateful-resources-managed.yaml
@@ -8,10 +8,11 @@ mcm-db:
     backup_window: 01:00-04:00
     db_name: mcm
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -27,12 +28,10 @@ mcm-db:
         value: utf8mb4
       - name: character_set_server
         value: utf8mb4
-      - name: sql_require_primary_key
-        value: OFF
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox
@@ -49,10 +48,11 @@ account-lookup-db:
     backup_window: 01:00-04:00
     db_name: account_lookup
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -71,7 +71,7 @@ account-lookup-db:
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox
@@ -88,10 +88,11 @@ central-ledger-db:
     backup_window: 01:00-04:00
     db_name: central_ledger
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -110,7 +111,7 @@ central-ledger-db:
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox
@@ -236,10 +237,11 @@ keycloak-db:
     backup_window: 01:00-04:00
     db_name: keycloak
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -258,7 +260,7 @@ keycloak-db:
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox
@@ -275,10 +277,11 @@ keto-db:
     backup_window: 01:00-04:00
     db_name: keto
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -297,7 +300,7 @@ keto-db:
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox
@@ -314,10 +317,11 @@ kratos-db:
     backup_window: 01:00-04:00
     db_name: kratos
     deletion_protection: false
-    engine: mysql
-    engine_version: "8.0"
-    family: mysql8.0
-    instance_class: db.m5d.large
+    engine: aurora-mysql
+    engine_version: 8.0.mysql_aurora.3.04.0
+    family: aurora-mysql8.0
+    instance_class: db.t3.medium
+    replicas: 1
     maintenance_window: Sun:04:00-Sun:06:00
     major_engine_version: "8.0"
     monitoring_interval: "30"
@@ -336,7 +340,7 @@ kratos-db:
     port: "3306"
     skip_final_snapshot: true
     multi_az: false
-    storage_encrypted: false
+    storage_encrypted: true
     tags:
       Origin: Terraform
       mojaloop/cost_center: mlf-iac-sandbox

diff --git a/terraform/k8s/default-config/platform-stateful-resources.yaml b/terraform/k8s/default-config/platform-stateful-resources.yaml
@@ -131,7 +131,7 @@ bulk-mongodb:
   enabled: true
   app_owner: "mojaloop"
   resource_type: mongodb
-  deployment_type: helm-chart
+  deployment_type: operator
   logical_service_config:
     logical_service_port: 27017
     logical_service_name: bulk-mongodb-svc
@@ -153,7 +153,7 @@ cep-mongodb:
   enabled: false
   app_owner: "mojaloop"
   resource_type: mongodb
-  deployment_type: helm-chart
+  deployment_type: operator
   logical_service_config:
     logical_service_port: 27017
     logical_service_name: mongodb-cep-mongodb
@@ -175,7 +175,7 @@ reporting-events-mongodb:
   enabled: true
   app_owner: "mojaloop"
   resource_type: mongodb
-  deployment_type: helm-chart
+  deployment_type: operator
   logical_service_config:
     logical_service_port: 27017
     logical_service_name: mongodb-reporting-events-mongodb
@@ -257,7 +257,7 @@ ttk-mongodb:
   enabled: true
   app_owner: "mojaloop"
   resource_type: mongodb
-  deployment_type: helm-chart
+  deployment_type: operator
   logical_service_config:
     logical_service_port: 27017
     logical_service_name: ttk-mongodb-svc