Refactor/MZ-574

.github/pull_request_template.md

@@ -3,8 +3,9 @@
 ## Pull Request Type
 [//]: # (Check the appropriate box for the type of pull request.)
 
-- [ ] Ledger
 - [ ] Auth
+- [ ] Infra
+- [ ] Ledger
 - [ ] Mdz
 - [ ] Transaction
 - [ ] Pipeline

Makefile

@@ -1,4 +1,5 @@
 AUTH_DIR := ./components/auth
+INFRA_DIR := ./components/infra
 LEDGER_DIR := ./components/ledger
 TRANSACTION_DIR := ./components/transaction
 
@@ -16,8 +17,10 @@ help:
 	@echo "    make format                              Run code formatter."
 	@echo "    make checkEnvs                           Check if github hooks are installed and secret env on files are not exposed."
 	@echo "    make auth                                Run a command inside the auth app in the components directory to see available commands."
+	@echo "    make infra                               Run a command inside the infra app in the components directory to see available commands."
 	@echo "    make ledger                              Run a command inside the ledger app in the components directory to see available commands."
 	@echo "    make transaction                         Run a command inside the transaction app in the components directory to see available commands."
+	@echo "    make set-env                             Run a command to copy all .env.example to .env into respective folders."
 	@echo "    make all-services                        Run a command to all services passing any individual container command."
 	@echo ""
 	@echo "  ## Utility Commands"
@@ -61,14 +64,18 @@ sec:
 	gosec ./...
 
 
-set_env:
+set-env:
 	cp -r $(AUTH_DIR)/.env.example $(AUTH_DIR)/.env
+	cp -r $(INFRA_DIR)/.env.example $(INFRA_DIR)/.env
 	cp -r $(LEDGER_DIR)/.env.example $(LEDGER_DIR)/.env
-	cp -r $(TRANSACTION_DIR)/.env.example $(TRANSACTION_DIR).env
+	cp -r $(TRANSACTION_DIR)/.env.example $(TRANSACTION_DIR)/.env
 
 auth:
 	$(MAKE) -C $(AUTH_DIR) $(COMMAND)
 
+infra:
+	$(MAKE) -C $(INFRA_DIR) $(COMMAND)
+
 ledger:
 	$(MAKE) -C $(LEDGER_DIR) $(COMMAND)
 
@@ -77,5 +84,6 @@ transaction:
 
 all-services:
 	$(MAKE) -C $(AUTH_DIR) $(COMMAND) && \
+	$(MAKE) -C $(INFRA_DIR) $(COMMAND) && \
 	$(MAKE) -C $(LEDGER_DIR) $(COMMAND) && \
 	$(MAKE) -C $(TRANSACTION_DIR) $(COMMAND)

common/mcasdoor/casdoor.go

@@ -2,10 +2,13 @@ package mcasdoor
 
 import (
 	_ "embed"
+	"encoding/json"
 	"errors"
 	"github.com/LerianStudio/midaz/common/mlog"
 	"github.com/casdoor/casdoor-go-sdk/casdoorsdk"
 	"go.uber.org/zap"
+	"io"
+	"net/http"
 )
 
 //go:embed certificates/token_jwt_key.pem
@@ -45,12 +48,16 @@ func (cc *CasdoorConnection) Connect() error {
 	}
 
 	client := casdoorsdk.NewClientWithConf(conf)
-	if client != nil {
-		cc.Logger.Info("Connected to casdoor ✅ \n")
+	if client == nil || !cc.healthCheck() {
+		cc.Connected = false
+		err := errors.New("can't connect casdoor")
+		cc.Logger.Fatalf("CasdoorConnection.Ping %v", zap.Error(err))
 
-		cc.Connected = true
+		return err
 	}
 
+	cc.Logger.Info("Connected to casdoor ✅ ")
+	cc.Connected = true
 	cc.Client = client
 
 	return nil
@@ -67,3 +74,39 @@ func (cc *CasdoorConnection) GetClient() (*casdoorsdk.Client, error) {
 
 	return cc.Client, nil
 }
+
+func (cc *CasdoorConnection) healthCheck() bool {
+	resp, err := http.Get(cc.Endpoint + "/api/health")
+
+	if err != nil {
+		cc.Logger.Errorf("failed to make GET request: %v", err.Error())
+
+		return false
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		cc.Logger.Errorf("failed to read response body: %v", err.Error())
+
+		return false
+	}
+
+	result := make(map[string]any)
+
+	err = json.Unmarshal(body, &result)
+	if err != nil {
+		cc.Logger.Errorf("failed to unmarshal response: %v", err.Error())
+
+		return false
+	}
+
+	if status, ok := result["status"].(string); ok && status == "ok" {
+		return true
+	}
+
+	cc.Logger.Error("casdoor unhealthy...")
+
+	return false
+}

common/mgrpc/grpc.go

@@ -2,6 +2,7 @@ package mgrpc
 
 import (
 	"fmt"
+	"github.com/LerianStudio/midaz/common/mlog"
 	"log"
 
 	"go.uber.org/zap"
@@ -11,8 +12,9 @@ import (
 
 // GRPCConnection is a struct which deal with gRPC connections.
 type GRPCConnection struct {
-	Addr string
-	Conn *grpc.ClientConn
+	Addr   string
+	Conn   *grpc.ClientConn
+	Logger mlog.Logger
 }
 
 // Connect keeps a singleton connection with gRPC.

components/auth/.env.example

@@ -9,7 +9,7 @@ DB_HOST=casdoor-db
 CASDOOR_DB_USER=midaz
 CASDOOR_DB_NAME=casdoor
 CASDOOR_DB_PASSWORD=leriand
-CASDOOR_DB_PORT=5436
+CASDOOR_DB_PORT=5700
 USER_EXECUTE_COMMAND=postgres
 
 # LOG

components/auth/docker-compose.yml

@@ -5,7 +5,7 @@ x-postgres-common:
   env_file:
     - .env
   networks:
-    - midaz_network
+    - auth_network
 
 services:
   casdoor:
@@ -21,7 +21,7 @@ services:
       casdoor-db:
         condition: service_healthy
     networks:
-      - midaz_network
+      - auth_network
 
   casdoor-db:
     <<: *postgres-common
@@ -43,6 +43,6 @@ services:
       retries: 5
 
 networks:
-  midaz_network:
-    name: midaz_network
+  auth_network:
+    name: auth_network
     driver: bridge

components/infra/.env.example

@@ -0,0 +1,25 @@
+# INFRA
+
+# DB POSTGRESQL PRIMARY AND REPLICA
+DB_HOST=primary-ledger
+DB_USER=midaz
+DB_PASSWORD=lerian
+DB_PORT=5701
+
+DB_REPLICA_HOST=replica-ledger
+DB_REPLICA_USER=midaz
+DB_REPLICA_PASSWORD=lerian
+DB_REPLICA_PORT=5702
+
+REPLICATION_USER=replicator
+REPLICATION_PASSWORD=replicator_password
+USER_EXECUTE_COMMAND=postgres
+
+# MONGO DB
+MONGO_HOST=mongodb
+MONGO_USER=midaz
+MONGO_PASSWORD=lerian
+MONGO_PORT=5703
+
+# REDIS
+REDIS_PORT=5704

components/infra/Makefile

@@ -0,0 +1,71 @@
+service_name := infra-service
+bin_dir := ./.bin
+artifacts_dir := ./artifacts
+
+$(shell mkdir -p $(artifacts_dir))
+
+.PHONY: help build up start down destroy stop restart logs logs-api ps login-timescale login-api db-shell
+
+# Display available commands
+info:
+	@echo "                                                                                                                                       "
+	@echo "                                                                                                                                       "
+	@echo "To run a specific command inside the infra container using make, you can execute:                                                     "
+	@echo "                                                                                                                                       "
+	@echo "make infra COMMAND=\"any\"                                                                                                            "
+	@echo "                                                                                                                                       "
+	@echo "This command will run the specified command inside the infra container. Replace \"any\" with the desired command you want to execute. "
+	@echo "                                                                                                                                       "
+	@echo "                                                                                                                         "
+	@echo "## Docker commands:"
+	@echo "                                                                                                                         "
+	@echo "  COMMAND=\"build\"                                Builds all Docker images defined in docker-compose.yml."
+	@echo "  COMMAND=\"up\"                                   Starts and runs all services defined in docker-compose.yml."
+	@echo "  COMMAND=\"start\"                                Starts existing containers defined in docker-compose.yml without creating them."
+	@echo "  COMMAND=\"stop\"                                 Stops running containers defined in docker-compose.yml without removing them."
+	@echo "  COMMAND=\"down\"                                 Stops and removes containers, networks, and volumes defined in docker-compose.yml."
+	@echo "  COMMAND=\"destroy\"                              Stops and removes containers, networks, and volumes (including named volumes) defined in docker-compose.yml."
+	@echo "  COMMAND=\"restart\"                              Stops and removes containers, networks, and volumes, then starts all services in detached mode."
+	@echo "  COMMAND=\"logs\"                                 Shows the last 100 lines of logs and follows live log output for services defined in docker-compose.yml."
+	@echo "  COMMAND=\"infra-api\"                             Shows the last 100 lines of logs and follows live log output for the infra service defined in docker-compose.yml."
+	@echo "  COMMAND=\"ps\"                                   Lists the status of containers defined in docker-compose.yml."
+	@echo "  COMMAND=\"db-shell\"                             Starts a PostgreSQL interactive terminal inside the infra container, logging in as the postgres user."
+
+
+# Docker Compose Commands
+build:
+	@docker-compose -f docker-compose.yml build
+
+up:
+	@docker-compose -f docker-compose.yml up -d
+
+start:
+	@docker-compose -f docker-compose.yml start
+
+down:
+	@docker-compose -f docker-compose.yml down
+
+destroy:
+	@docker-compose -f docker-compose.yml down -v
+
+stop:
+	@docker-compose -f docker-compose.yml stop
+
+restart:
+	docker-compose -f docker-compose.yml down && \
+    docker-compose -f docker-compose.yml up -d
+
+logs:
+	@docker-compose -f docker-compose.yml logs --tail=100 -f
+
+logs-api:
+	@docker-compose -f docker-compose.yml logs --tail=100 -f infra
+
+ps:
+	@docker-compose -f docker-compose.yml ps
+
+infra-api:
+	@docker-compose -f docker-compose.yml exec infra /bin/bash
+
+db-shell:
+	@docker-compose -f docker-compose.yml exec infra psql -Upostgres

components/infra/docker-compose.yml

@@ -0,0 +1,113 @@
+x-postgres-ledger-common:
+  &postgres-ledger-common
+  image: postgres:16-alpine
+  user: ${USER_EXECUTE_COMMAND}
+  restart: always
+  healthcheck:
+    test: [ "CMD-SHELL", "pg_isready -U ${DB_USER} -p ${DB_PORT}" ]
+    interval: 10s
+    timeout: 5s
+    retries: 5
+  networks:
+    - infra_network
+
+x-mongodb-common:
+  &mongodb-common
+  image: mongo:latest
+  restart: always
+  healthcheck:
+    test: echo 'db.runCommand("ping").ok'
+    interval: 10s
+    timeout: 5s
+    retries: 5
+  networks:
+    - infra_network
+
+x-redis-common:
+  &redis-common
+  image: redis:latest
+  env_file:
+    - .env
+  networks:
+    - infra_network
+
+services:
+  mongodb:
+    <<: *mongodb-common
+    container_name: midaz-mongodb
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD}
+    command: mongod --port ${MONGO_PORT}
+    ports:
+      - ${MONGO_PORT}:${MONGO_PORT}
+    volumes:
+      - mongodb_data_container:/data/db
+
+  redis:
+    <<: *redis-common
+    container_name: midaz-redis
+    ports:
+      - ${REDIS_PORT}:${REDIS_PORT}
+
+  primary-ledger:
+    <<: *postgres-ledger-common
+    container_name: midaz-postgres-primary
+    ports:
+      - ${DB_PORT}:${DB_PORT}
+    environment:
+      PGPORT: ${DB_PORT}
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256\nhost replication all 0.0.0.0/0 md5"
+      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
+    command: |
+      postgres
+      -c wal_level=replica
+      -c hot_standby=on
+      -c max_wal_senders=10
+      -c max_replication_slots=10
+      -c hot_standby_feedback=on
+    volumes:
+      - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  replica-ledger:
+    <<: *postgres-ledger-common
+    container_name: midaz-postgres-replica
+    ports:
+      - ${DB_REPLICA_PORT}:${DB_REPLICA_PORT}
+    environment:
+      PGPORT: ${DB_REPLICA_PORT}
+      PGUSER: ${REPLICATION_USER}
+      PGPASSWORD: ${REPLICATION_PASSWORD}
+    command: |
+      bash -c "
+      if [ ! -d \"/var/lib/postgresql/data\" ] || [ ! -f \"/var/lib/postgresql/data/postgresql.conf\" ]; then
+        until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=primary-ledger --port=${DB_PORT}
+        do
+          echo 'Waiting for primary-ledger to connect...'
+          sleep 1s
+        done
+        echo 'Backup done..., starting replica-ledger...'
+        chmod 0700 /var/lib/postgresql/data
+        # Ensure the port is set to use for the replica
+        sed -i 's/^#port.*/port = ${DB_REPLICA_PORT}/' /var/lib/postgresql/data/postgresql.conf
+      fi
+      exec postgres -c config_file=/var/lib/postgresql/data/postgresql.conf
+      "
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U ${DB_REPLICA_USER} -p ${DB_REPLICA_PORT}" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    depends_on:
+      primary-ledger:
+        condition: service_healthy
+
+volumes:
+  mongodb_data_container:
+
+networks:
+  infra_network:
+    name: infra_network
+    driver: bridge